How to pass 2 loaders to training of a multi input network

This is my training function:

def train(net, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
    # try:
      net = net.to(dev)
      #print(net)
      #summary(net,[(net.in_channels,net.in_width,net.in_height)]*2)


      criterion.to(dev)


      # Initialize history
      history_loss = {"train": [], "val": [], "test": []}
      history_accuracy = {"train": [], "val": [], "test": []}
      # Store the best val accuracy
      best_val_accuracy = 0

      # Process each epoch
      for epoch in range(epochs):
        # Initialize epoch variables
        sum_loss = {"train": 0, "val": 0, "test": 0}
        sum_accuracy = {"train": 0, "val": 0, "test": 0}

        progbar = None
        # Process each split
        for split in ["train", "val", "test"]:
          if split == "train":
            net.train()
            widgets = [
              ' [', pb.Timer(), '] ',
              pb.Bar(),
              ' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
            ]

            progbar = pb.ProgressBar(max_value=len(loaders[split]),widgets=widgets,redirect_stdout=True)

          else:
            net.eval()
          # Process each batch
          for j,(input, labels) in enumerate(loaders[split]):
            load_a = enumerate(loaders_a[split])
            load_b = enumerate(loaders_b[split])
            labels = labels.unsqueeze(1)
            labels = labels.float()

            input = input.to(dev)
            labels = labels.to(dev)

            _,(input_a, labels_a) = load_a[j]
            _,(input_b, labels_b) = load_b[j]

            input_a = input.to(dev)
            labels_a = labels.to(dev)
            input_b = input.to(dev)
            labels_b = labels.to(dev)

            # Reset gradients
            optimizer.zero_grad()
            # Compute output
            pred = net(input_a,input_b)

            loss = criterion(pred, [labels_a, labels_b])
            # Update loss
            sum_loss[split] += loss.item()
            # Check parameter update
            if split == "train":
              # Compute gradients
              loss.backward()
              # Optimize
              optimizer.step()

            # Compute accuracy

            #https://discuss.pytorch.org/t/bcewithlogitsloss-and-model-accuracy-calculation/59293/ 2
            pred_labels = (pred[2] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_a = (pred[0] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_b = (pred[1] >= 0.0).long()  # Binarize predictions to 0 and 1


            batch_accuracy = (pred_labels == labels).sum().item() / len(labels)
            batch_accuracy_a = (pred_labels_a == labels_a).sum().item() / len(labels_a)
            batch_accuracy_b = (pred_labels_b == labels_b).sum().item() / len(labels_b)
            # Update accuracy
            sum_accuracy[split] += batch_accuracy

            if (split=='train'):
              progbar.update(j, ta=batch_accuracy)

        if (progbar is not None):
          progbar.finish()
        # Compute epoch loss/accuracy
        epoch_loss = {split: sum_loss[split] / len(loaders[split]) for split in ["train", "val", "test"]}
        epoch_accuracy = {split: sum_accuracy[split] / len(loaders[split]) for split in ["train", "val", "test"]}

        # Store params at the best validation accuracy
        if save_param and epoch_accuracy["val"] > best_val_accuracy:
          # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
          torch.save(net.state_dict(), f"{model_name}_best_val.pth")
          best_val_accuracy = epoch_accuracy["val"]

        # Update history
        for split in ["train", "val", "test"]:
          history_loss[split].append(epoch_loss[split])
          history_accuracy[split].append(epoch_accuracy[split])
        # Print info
        print(f"Epoch {epoch + 1}:",
              f"TrL={epoch_loss['train']:.4f},",
              f"TrA={epoch_accuracy['train']:.4f},",
              f"VL={epoch_loss['val']:.4f},",
              f"VA={epoch_accuracy['val']:.4f},",
              f"TeL={epoch_loss['test']:.4f},",
              f"TeA={epoch_accuracy['test']:.4f},")
    # except KeyboardInterrupt:
    #   print("Interrupted")
    # finally:
    #   # Plot loss
    #   plt.title("Loss")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_loss[split], label=split)
    #   plt.legend()
    #   plt.show()
    #   # Plot accuracy
    #   plt.title("Accuracy")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_accuracy[split], label=split)
    #   plt.legend()
    #   plt.show()

I have 2 data loaders, loaders_a and loaders_b, because my net is multi input. How can I modify this training function in order to work with these 2 loaders?
These are the loaders:

# Define dictionary of loaders
loaders = {"train": train_loader,
           "val": val_loader,
           "test": test_loader}

# Define dictionary of loaders
loaders_a = {"train": train_loader_a,
           "val": val_loader_a,
           "test": test_loader_a}

loaders_b = {"train": train_loader_b,
           "val": val_loader_b,
           "test": test_loader_b}

Wha’ts wrong with the solution we’ve talked about some time ago?

I have changed the entire structure of the code. I am really confused on how to make it works correctly.
I am sure that the network is implemented in the right way, but I have to modify this damned training function.
I have always billions of errors… " ValueError: too many values to unpack (expected 2)", “TypeError: tuple indices must be integers or slices, not str”,…

I would try:

for (inp1, tgt1), (inp2, tgt2) in zip(loader1[split], loader2[split])

Like this?

def train(net, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
    # try:
      net = net.to(dev)
      #print(net)
      #summary(net,[(net.in_channels,net.in_width,net.in_height)]*2)


      criterion.to(dev)


      # Initialize history
      history_loss = {"train": [], "val": [], "test": []}
      history_accuracy_a = {"train": [], "val": [], "test": []}
      history_accuracy_b = {"train": [], "val": [], "test": []}
      # Store the best val accuracy
      best_val_accuracy = 0

      # Process each epoch
      for epoch in range(epochs):
        # Initialize epoch variables
        sum_loss = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_a = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_b = {"train": 0, "val": 0, "test": 0}

        progbar = None
        # Process each split
        for split in ["train", "val", "test"]:
          if split == "train":
            net.train()
            widgets = [
              ' [', pb.Timer(), '] ',
              pb.Bar(),
              ' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
            ]

            progbar = pb.ProgressBar(max_value=len(loaders_a[split]),widgets=widgets,redirect_stdout=True)

          else:
            net.eval()
          # Process each batch
          j=0
          for ((input_a, labels_a), (input_b, labels_b)) in zip(loaders_a[split], loaders_b[split]):
            labels_a = labels_a.unsqueeze(1)
            labels_a = labels_a.float()
            labels_b = labels_b.unsqueeze(1)
            labels_b = labels_b.float()
            #labels = torch.cat(labels_a, labels_b)
            #labels = labels.unsqueeze(1)
            #labels = labels.float()

            #input = input.to(dev)
            #labels = labels.to(dev)
            input_a = input_a.to(dev)
            labels_a = labels_a.to(dev)
            input_b = input_b.to(dev)
            labels_b = labels_b.to(dev)

            # Reset gradients
            optimizer.zero_grad()
            # Compute output
            pred = net(input_a,input_b)

            loss = criterion(pred, [labels_a, labels_b])
            # Update loss
            sum_loss[split] += loss.item()
            # Check parameter update
            if split == "train":
              # Compute gradients
              loss.backward()
              # Optimize
              optimizer.step()

            # Compute accuracy
            #pred_labels = (pred[2] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_a = (pred[0] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_b = (pred[1] >= 0.0).long()  # Binarize predictions to 0 and 1


            #batch_accuracy = (pred_labels == labels).sum().item() / len(labels)
            batch_accuracy_a = (pred_labels_a == labels_a).sum().item() / len(labels_a)
            batch_accuracy_b = (pred_labels_b == labels_b).sum().item() / len(labels_b)
            # Update accurac
            #sum_accuracy[split] += batch_accuracy
            sum_accuracy_a[split] += batch_accuracy_a
            sum_accuracy_b[split] += batch_accuracy_b

            if (split=='train'):
              #progbar.update(j, ta=batch_accuracy)
              progbar.update(j, ta=batch_accuracy_a)
              progbar.update(j, ta=batch_accuracy_b)
            
            j=j+1

        if (progbar is not None):
          progbar.finish()
        # Compute epoch loss/accuracy
        epoch_loss = {split: sum_loss[split] / len(loaders[split]) for split in ["train", "val", "test"]}
        #epoch_accuracy = {split: sum_accuracy[split] / len(loaders[split]) for split in ["train", "val", "test"]}
        epoch_accuracy_a = {split: sum_accuracy_a[split] / len(loaders_a[split]) for split in ["train", "val", "test"]}
        epoch_accuracy_b = {split: sum_accuracy_b[split] / len(loaders_b[split]) for split in ["train", "val", "test"]}

        # Store params at the best validation accuracy
        if save_param and epoch_accuracy["val"] > best_val_accuracy:
          # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
          torch.save(net.state_dict(), f"{model_name}_best_val.pth")
          best_val_accuracy = epoch_accuracy["val"]

        # Update history
        for split in ["train", "val", "test"]:
          history_loss[split].append(epoch_loss[split])
          history_accuracy_a[split].append(epoch_accuracy_a[split])
          history_accuracy_b[split].append(epoch_accuracy_b[split])
        # Print info
        print(f"Epoch {epoch + 1}:",
              f"TrL={epoch_loss['train']:.4f},",
              f"TrA={epoch_accuracy_a['train']:.4f},",
              f"VL={epoch_loss['val']:.4f},",
              f"VA={epoch_accuracy_a['val']:.4f},",
              f"TeL={epoch_loss['test']:.4f},",
              f"TeA={epoch_accuracy_a['test']:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"TrL={epoch_loss['train']:.4f},",
              f"TrA={epoch_accuracy_b['train']:.4f},",
              f"VL={epoch_loss['val']:.4f},",
              f"VA={epoch_accuracy_b['val']:.4f},",
              f"TeL={epoch_loss['test']:.4f},",
              f"TeA={epoch_accuracy_b['test']:.4f},")
    # except KeyboardInterrupt:
    #   print("Interrupted")
    # finally:
    #   # Plot loss
    #   plt.title("Loss")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_loss[split], label=split)
    #   plt.legend()
    #   plt.show()
    #   # Plot accuracy
    #   plt.title("Accuracy")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_accuracy[split], label=split)
    #   plt.legend()
    #   plt.show()

Train model

train(combo, (loaders_a, loaders_b), optimizer, criterion, epochs=10, dev=dev)

Can’t see where the loaders_a and loaders_b variables are made.

I would consider the loaders argument of this function to accept a list of loaders (or tuple).

Then, inside the function, just unpack:
loaders_a, loaders_b = loaders

I wrote loaders_a and loaders_b in the first message
You say, in this way?

def train(net, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
   loaders_a, loaders_b = loaders
    # try:
      net = net.to(dev)
      #print(net)
      #summary(net,[(net.in_channels,net.in_width,net.in_height)]*2)


      criterion.to(dev)


      # Initialize history
      history_loss = {"train": [], "val": [], "test": []}
      history_accuracy_a = {"train": [], "val": [], "test": []}
      history_accuracy_b = {"train": [], "val": [], "test": []}
      # Store the best val accuracy
      best_val_accuracy = 0

      # Process each epoch
      for epoch in range(epochs):
        # Initialize epoch variables
        sum_loss = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_a = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_b = {"train": 0, "val": 0, "test": 0}

        progbar = None
        # Process each split
        for split in ["train", "val", "test"]:
          if split == "train":
            net.train()
            widgets = [
              ' [', pb.Timer(), '] ',
              pb.Bar(),
              ' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
            ]

            progbar = pb.ProgressBar(max_value=len(loaders_a[split]),widgets=widgets,redirect_stdout=True)

          else:
            net.eval()
          # Process each batch
          j=0
          for ((input_a, labels_a), (input_b, labels_b)) in zip(loaders_a[split], loaders_b[split]):
            labels_a = labels_a.unsqueeze(1)
            labels_a = labels_a.float()
            labels_b = labels_b.unsqueeze(1)
            labels_b = labels_b.float()
            #labels = torch.cat(labels_a, labels_b)
            #labels = labels.unsqueeze(1)
            #labels = labels.float()

            #input = input.to(dev)
            #labels = labels.to(dev)
            input_a = input_a.to(dev)
            labels_a = labels_a.to(dev)
            input_b = input_b.to(dev)
            labels_b = labels_b.to(dev)

            # Reset gradients
            optimizer.zero_grad()
            # Compute output
            pred = net(input_a,input_b)

            loss = criterion(pred, [labels_a, labels_b])
            # Update loss
            sum_loss[split] += loss.item()
            # Check parameter update
            if split == "train":
              # Compute gradients
              loss.backward()
              # Optimize
              optimizer.step()

            # Compute accuracy
            #pred_labels = (pred[2] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_a = (pred[0] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_b = (pred[1] >= 0.0).long()  # Binarize predictions to 0 and 1


            #batch_accuracy = (pred_labels == labels).sum().item() / len(labels)
            batch_accuracy_a = (pred_labels_a == labels_a).sum().item() / len(labels_a)
            batch_accuracy_b = (pred_labels_b == labels_b).sum().item() / len(labels_b)
            # Update accurac
            #sum_accuracy[split] += batch_accuracy
            sum_accuracy_a[split] += batch_accuracy_a
            sum_accuracy_b[split] += batch_accuracy_b

            if (split=='train'):
              #progbar.update(j, ta=batch_accuracy)
              progbar.update(j, ta=batch_accuracy_a)
              progbar.update(j, ta=batch_accuracy_b)
            
            j=j+1

        if (progbar is not None):
          progbar.finish()
        # Compute epoch loss/accuracy
        epoch_loss = {split: sum_loss[split] / len(loaders[split]) for split in ["train", "val", "test"]}
        #epoch_accuracy = {split: sum_accuracy[split] / len(loaders[split]) for split in ["train", "val", "test"]}
        epoch_accuracy_a = {split: sum_accuracy_a[split] / len(loaders_a[split]) for split in ["train", "val", "test"]}
        epoch_accuracy_b = {split: sum_accuracy_b[split] / len(loaders_b[split]) for split in ["train", "val", "test"]}

        # Store params at the best validation accuracy
        if save_param and epoch_accuracy["val"] > best_val_accuracy:
          # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
          torch.save(net.state_dict(), f"{model_name}_best_val.pth")
          best_val_accuracy = epoch_accuracy["val"]

        # Update history
        for split in ["train", "val", "test"]:
          history_loss[split].append(epoch_loss[split])
          history_accuracy_a[split].append(epoch_accuracy_a[split])
          history_accuracy_b[split].append(epoch_accuracy_b[split])
        # Print info
        print(f"Epoch {epoch + 1}:",
              f"TrL={epoch_loss['train']:.4f},",
              f"TrA={epoch_accuracy_a['train']:.4f},",
              f"VL={epoch_loss['val']:.4f},",
              f"VA={epoch_accuracy_a['val']:.4f},",
              f"TeL={epoch_loss['test']:.4f},",
              f"TeA={epoch_accuracy_a['test']:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"TrL={epoch_loss['train']:.4f},",
              f"TrA={epoch_accuracy_b['train']:.4f},",
              f"VL={epoch_loss['val']:.4f},",
              f"VA={epoch_accuracy_b['val']:.4f},",
              f"TeL={epoch_loss['test']:.4f},",
              f"TeA={epoch_accuracy_b['test']:.4f},")
    # except KeyboardInterrupt:
    #   print("Interrupted")
    # finally:
    #   # Plot loss
    #   plt.title("Loss")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_loss[split], label=split)
    #   plt.legend()
    #   plt.show()
    #   # Plot accuracy
    #   plt.title("Accuracy")
    #   for split in ["train", "val", "test"]:
    #     plt.plot(history_accuracy[split], label=split)
    #   plt.legend()
    #   plt.show()

And

train(combo, (loaders_a, loaders_b), optimizer, criterion, epochs=10, dev=dev)

Yep, should work.

BTW, why are you “splitting” the datasets into several subsets?
I think it would be much easier to just make arguments train_loaders and val_loaders.

And the actual tests should be performed after the training has finished. It doesn’t make sense to have two subsets that essentially do the same during training.

You seem to have ventured into something called “spaghetti code” - it does so much things, that it becomes a problem to change one thing, because dozen errors happen instantly if you do.

First of all, with the previous solution of training, it seems it works, but after 20-30 seconds of work, it prints an error:

IndexError: list index out of range

I have not understood the problem you said then.
I have 2 datasets, and for each dataset I have train, val and test (both sets and dataloaders)
And finally I create a vocabulary called loader_a, loader_b and loader

Which line seems to cause the error?
(might be hard to locate - try to print some debug info like “training finished” etc).

It seems this line:

---> 43           for ((input_a, labels_a), (input_b, labels_b)) in zip(loaders_a[split], loaders_b[split]):

I put these 2 print (look some lines before the end of this code):

def train(net, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
      loaders_a, loaders_b = loaders
    # try:
      net = net.to(dev)
      #print(net)
      #summary(net,[(net.in_channels,net.in_width,net.in_height)]*2)


      criterion.to(dev)


      # Initialize history
      history_loss = {"train": [], "val": [], "test": []}
      history_accuracy_a = {"train": [], "val": [], "test": []}
      history_accuracy_b = {"train": [], "val": [], "test": []}
      # Store the best val accuracy
      best_val_accuracy = 0

      # Process each epoch
      for epoch in range(epochs):
        # Initialize epoch variables
        sum_loss = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_a = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_b = {"train": 0, "val": 0, "test": 0}

        progbar = None
        # Process each split
        for split in ["train", "val", "test"]:
          if split == "train":
            net.train()
            widgets = [
              ' [', pb.Timer(), '] ',
              pb.Bar(),
              ' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
            ]

            progbar = pb.ProgressBar(max_value=len(loaders_a[split]),widgets=widgets,redirect_stdout=True)

          else:
            net.eval()
            print("Evaluation")
          # Process each batch
          print("Pre-for")
          j=0
          for ((input_a, labels_a), (input_b, labels_b)) in zip(loaders_a[split], loaders_b[split]):

The output is only this (none of the 2 prints I wrote):

 [Elapsed Time: 0:02:14] |############## | [ETA:   0:00:00] [Train Acc:  0.992]

Try to print somewhere inside this for, I doubt this line would raise such error.

Also the lack of pre-for text indicates that the error might be somewhere above. Or you just forgot to update the function code by running the cell again :stuck_out_tongue:

I added these 2 prints:

for ((input_a, labels_a), (input_b, labels_b)) in zip(loaders_a[split], loaders_b[split]):
            print("first print inside for")
            labels_a = labels_a.unsqueeze(1)
            labels_a = labels_a.float()
            labels_b = labels_b.unsqueeze(1)
            labels_b = labels_b.float()
            #labels = torch.cat(labels_a, labels_b)
            #labels = labels.unsqueeze(1)
            #labels = labels.float()

            #input = input.to(dev)
            #labels = labels.to(dev)
            input_a = input_a.to(dev)
            labels_a = labels_a.to(dev)
            input_b = input_b.to(dev)
            labels_b = labels_b.to(dev)

            # Reset gradients
            optimizer.zero_grad()
            # Compute output
            pred = net(input_a,input_b)

            loss = criterion(pred, [labels_a, labels_b])
            # Update loss
            sum_loss[split] += loss.item()
            # Check parameter update
            if split == "train":
              # Compute gradients
              loss.backward()
              # Optimize
              optimizer.step()
              print("second print inside for")

            # Compute accuracy

And the output is only this ;(
[Elapsed Time: 0:13:51] |############## | [ETA: 0:00:04] [Train Acc: 0.992]

Make sure you really run the cell with train function again, so the prints get included in the execution.

No reason for sad faces, we all go through similar phases :stuck_out_tongue:

BTW, does it really take 13 seconds/minutes to reach the point where it fails? This seems like a value out of nowhere.

No, the time is wrong, I don’t know why.
Timer is sometimes right, sometimes wrong: it works like this: 1s, 2s, 3s, 4s, 13minute, 6s, 7s, 8s, 35 minutes, 10s, 11s, 6 minutes :smiley:
There is some problem on the function of the timer.
However, it is not important.
I obviously run again the cell with the definition of the train function. But I have that error…