A detailed example of data loaders with PyTorch

PHOTO EMBED

Wed Jul 14 2021 15:13:44 GMT+0000 (Coordinated Universal Time)

Saved by @QuinnFox12 #textpreprocessing #nlp #gnn #dataload #pytorch

# Load entire dataset
X, y = torch.load('some_training_set_with_labels.pt')
 
# Train model
for epoch in range(max_epochs):
    for i in range(n_batches):
        # Local batches and labels
        local_X, local_y = X[i*n_batches:(i+1)*n_batches,], y[i*n_batches:(i+1)*n_batches,]
 
        # Your model
        [...]
         
         
# other
# Unoptimized generator
training_generator = SomeSingleCoreGenerator('some_training_set_with_labels.pt')
 
# Train model
for epoch in range(max_epochs):
    for local_X, local_y in training_generator:
        # Your model
        [...]
content_copyCOPY