Skip to content
Snippets Groups Projects
Commit b8360315 authored by MARMORET Axel's avatar MARMORET Axel
Browse files

Adding conversion functions, between tensors, barwise TF and spectrograms

parent 49df6802
No related branches found
No related tags found
No related merge requests found
......@@ -29,9 +29,9 @@ import numpy as np
import tensorly as tl
import librosa
# %% Tensors barwise spectrograms construction
# %% Spectrograms to tensors
# !!! Be extremely careful with the organization of modes, which can be either Frequency-Time at barscale-Bars (FTB) or Bars-Frequency-Time at barscale (BFT) depending on the method.
def tensorize_barwise_BFT(spectrogram, bars, hop_length_seconds, subdivision):
def tensorize_barwise_BFT(spectrogram, bars, hop_length_seconds, subdivision, subset_nb_bars = None):
"""
Returns a 3rd order tensor-spectrogram from the original spectrogram and bars starts and ends.
The order of modes is Bars-Frequency-Time at barscale (BFT).
......@@ -60,17 +60,20 @@ def tensorize_barwise_BFT(spectrogram, bars, hop_length_seconds, subdivision):
"""
barwise_spec = []
bars_idx = dm.segments_from_time_to_frame_idx(bars[1:], hop_length_seconds)
if subset_nb_bars is not None:
bars_idx = bars_idx[:subset_nb_bars]
for idx, beats in enumerate(bars_idx):
t_0 = beats[0]
t_1 = beats[1]
samples = [int(round(t_0 + k * (t_1 - t_0)/subdivision)) for k in range(subdivision)]
if len(samples) != len(set(samples)): # Check for repetitions
if idx != len(bars_idx) - 1: # It's not a problem if it's the last bar, because it is due to inconsistencies between the last downbeat estimated and the end of the song.
raise err.ToDebugException("The subdivision is too large, it leads to repeated samples chosen in the bar!")
if samples[-1] < spectrogram.shape[1]:
barwise_spec.append(spectrogram[:,samples])
return np.array(barwise_spec)
def tensorize_barwise_FTB(spectrogram, bars, hop_length_seconds, subdivision):
def tensorize_barwise_FTB(spectrogram, bars, hop_length_seconds, subdivision, subset_nb_bars = None):
#(careful: different mode organization than previous one: here, this is Frequency-Time-Bars)
"""
Returns a 3rd order tensor-spectrogram from the original spectrogram and bars starts and ends.
......@@ -100,6 +103,8 @@ def tensorize_barwise_FTB(spectrogram, bars, hop_length_seconds, subdivision):
"""
freq_len = spectrogram.shape[0]
bars_idx = dm.segments_from_time_to_frame_idx(bars[1:], hop_length_seconds)
if subset_nb_bars is not None:
bars_idx = bars_idx[:subset_nb_bars]
samples_init = [int(round(bars_idx[0][0] + k * (bars_idx[0][1] - bars_idx[0][0])/subdivision)) for k in range(subdivision)]
tens = np.array(spectrogram[:,samples_init]).reshape(freq_len, subdivision, 1)
......@@ -116,8 +121,42 @@ def tensorize_barwise_FTB(spectrogram, bars, hop_length_seconds, subdivision):
return tl.tensor(tens)#, dtype=tl.float32)
# %% Matrix barwise spectrograms handling
def barwise_TF_matrix(spectrogram, bars, hop_length_seconds, subdivision):
# %% Tensors to spectrograms
def tensor_barwise_to_spectrogram(tensor, mode_order = "BFT", subset_nb_bars = None):
if subset_nb_bars is not None:
tensor = barwise_subset_this_tensor(tensor, subset_nb_bars, mode_order = mode_order)
if mode_order == "BFT":
return tl.unfold(tensor, 1)
elif mode_order == "FTB":
return np.reshape(tensor, (tensor.shape[0], tensor.shape[1] * tensor.shape[2]), order = 'F') # Note: it is NOT the same than unfold(0)
else:
raise err.InvalidArgumentValueException(f"Unknown mode order: {mode_order}.")
def barwise_subset_this_tensor(tensor, subset_nb_bars, mode_order = "BFT"):
if mode_order == "BFT":
return tensor[:subset_nb_bars]
elif mode_order == "FTB":
return tensor[:,:,:subset_nb_bars]
else:
raise err.InvalidArgumentValueException(f"Unknown mode order: {mode_order}.")
def get_this_bar_tensor(tensor, bar_idx, mode_order = "BFT"):
if mode_order == "BFT":
return tensor[bar_idx]
elif mode_order == "FTB":
return tensor[:,:,bar_idx]
else:
raise err.InvalidArgumentValueException(f"Unknown mode order: {mode_order}.")
# %% Spectrogram to Barwise TF matrix
def barwise_TF_matrix(spectrogram, bars, hop_length_seconds, subdivision, subset_nb_bars = None):
"""
Barwise TF matrix, a 2D representation of Barwise spectrograms as Time-Frequency vectors.
See [1] for details.
......@@ -139,11 +178,15 @@ def barwise_TF_matrix(spectrogram, bars, hop_length_seconds, subdivision):
The Barwise TF matrix, of sizes (b, tf).
"""
tensor_spectrogram = tensorize_barwise_BFT(spectrogram, bars, hop_length_seconds, subdivision)
tensor_spectrogram = tensorize_barwise_BFT(spectrogram, bars, hop_length_seconds, subdivision, subset_nb_bars=subset_nb_bars)
return tl.unfold(tensor_spectrogram, 0)
# %% Vector barwise spectrogram handling
def TF_vector_to_TF_matrix(vector, frequency_dimension, subdivision):
def barwise_subset_this_TF_matrix(matrix, subset_nb_bars):
assert subset_nb_bars is not None
return matrix[:subset_nb_bars]
# %% Vector and Barwise TF to spectrogram
def TF_vector_to_spectrogram(vector, frequency_dimension, subdivision):
"""
Encapsulating the conversion from a Time-Frequency vector to a Time-Frequency matrix (spectrogram)
......@@ -167,7 +210,30 @@ def TF_vector_to_TF_matrix(vector, frequency_dimension, subdivision):
assert frequency_dimension*subdivision == vector.shape[0]
return tl.fold(vector, 0, (frequency_dimension,subdivision))
def TF_matrix_to_spectrogram(matrix, frequency_dimension, subdivision, subset_nb_bars = None):
spectrogram_content = None
if subset_nb_bars is not None:
matrix = barwise_subset_this_TF_matrix(matrix, subset_nb_bars)
for tf_vector in matrix:
bar_content = TF_vector_to_spectrogram(tf_vector, frequency_dimension, subdivision)
spectrogram_content = np.concatenate((spectrogram_content, bar_content), axis=1) if spectrogram_content is not None else bar_content
return spectrogram_content
# Tensor to Barwise TF
def tensor_barwise_to_barwise_TF(tensor, mode_order = "BFT"):
# Barmode: 0 for BTF, 2 for FTB
if mode_order == "BFT": # Checked
return tl.unfold(tensor, 0)
elif mode_order == "FTB": # Checked
return tl.unfold(tensor, 2)
else:
raise err.InvalidArgumentValueException(f"Unknown mode order: {mode_order}.")
# %% Barwise TF to tensor
# TODO
# Beatwise MSAF
def beat_synchronize_msaf(spectrogram, frame_times, beat_frames, beat_times):
# Make beat synchronous
beatsync_feats = librosa.util.utils.sync(spectrogram.T, beat_frames, pad=True).T
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment