# MIT License # # Copyright (c) 2026 audio-embeddings contributors # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from __future__ import annotations from typing import Sequence import torch from einops import rearrange from einops.layers.torch import Rearrange from torch import nn def _parse_conv_layers_spec( conv_layers_spec: str | Sequence[Sequence[int]] | Sequence[tuple[int, int, int]], ) -> list[tuple[int, int, int]]: if isinstance(conv_layers_spec, str): # Config-driven expression style used by wavjepa, e.g. # "[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512, 2, 2)]" parsed = eval(conv_layers_spec, {"__builtins__": {}}, {}) # noqa: S307 else: parsed = conv_layers_spec out: list[tuple[int, int, int]] = [] for layer in parsed: if len(layer) != 3: raise ValueError(f"Invalid conv layer spec {layer}, expected (dim, k, s)") dim, kernel, stride = layer out.append((int(dim), int(kernel), int(stride))) if len(out) == 0: raise ValueError("conv_layers_spec must contain at least one layer") return out class WaveformFeatureEncoder(nn.Module): """ Convolutional waveform feature encoder that outputs a token sequence. Input shape: [B, C, T] Output shape: [B, N, F] """ def __init__( self, conv_layers_spec: str | Sequence[Sequence[int]] | Sequence[ tuple[int, int, int] ] = "[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512, 2, 2)]", in_channels: int = 1, dropout: float = 0.0, mode: str = "default", conv_bias: bool = False, depthwise: bool = False, ) -> None: super().__init__() if mode not in {"default", "layer_norm"}: raise ValueError( f"Unknown mode='{mode}', expected 'default' or 'layer_norm'" ) self.conv_layers_spec = _parse_conv_layers_spec(conv_layers_spec) self.in_channels = in_channels self.depthwise = depthwise layers: list[nn.Module] = [] in_dim = in_channels for idx, (out_dim, kernel, stride) in enumerate(self.conv_layers_spec): layers.append( self._make_block( in_dim=in_dim, out_dim=out_dim, kernel=kernel, stride=stride, dropout=dropout, mode=mode, conv_bias=conv_bias, depthwise=depthwise, is_first=idx == 0, ) ) in_dim = out_dim self.cnn = nn.Sequential(*layers) self.embedding_dim = self.conv_layers_spec[-1][0] @staticmethod def _make_block( in_dim: int, out_dim: int, kernel: int, stride: int, dropout: float, mode: str, conv_bias: bool, depthwise: bool, is_first: bool, ) -> nn.Module: if depthwise: if out_dim % in_dim != 0: raise ValueError( "Depthwise mode requires out_dim to be a multiple of in_dim, " f"got out_dim={out_dim}, in_dim={in_dim}" ) conv = nn.Conv1d( in_dim, out_dim, kernel_size=kernel, stride=stride, bias=conv_bias, groups=in_dim, ) else: conv = nn.Conv1d( in_dim, out_dim, kernel_size=kernel, stride=stride, bias=conv_bias, ) nn.init.kaiming_normal_(conv.weight) if mode == "layer_norm": return nn.Sequential( conv, nn.Dropout(p=dropout), Rearrange("... c t -> ... t c"), nn.LayerNorm(out_dim, elementwise_affine=True), Rearrange("... t c -> ... c t"), nn.GELU(), ) if mode == "default" and is_first: return nn.Sequential( conv, nn.Dropout(p=dropout), nn.GroupNorm(out_dim, out_dim, affine=True), nn.GELU(), ) return nn.Sequential(conv, nn.Dropout(p=dropout), nn.GELU()) def forward(self, x: torch.Tensor) -> torch.Tensor: x = self.cnn(x) return rearrange(x, "b f n -> b n f") def total_patches(self, time_samples: int) -> int: n = int(time_samples) for _, kernel, stride in self.conv_layers_spec: if n < kernel: return 0 n = (n - kernel) // stride + 1 return int(n)