# Load entire dataset
X, y = torch.load('some_training_set_with_labels.pt')
# Train model
for epoch in range(max_epochs):
for i in range(n_batches):
# Local batches and labels
local_X, local_y = X[i*n_batches:(i+1)*n_batches,], y[i*n_batches:(i+1)*n_batches,]
# Your model
[...]
# other
# Unoptimized generator
training_generator = SomeSingleCoreGenerator('some_training_set_with_labels.pt')
# Train model
for epoch in range(max_epochs):
for local_X, local_y in training_generator:
# Your model
[...]