workflow w {
  File fileOfFilenames # 1GB in size

  #Split large file into small individual files
  call splitFile { input: largeFile = fileOfFilenames }

 scatter (f in splitFile.tiny_files) {
    String fileName = read_string(f)   
  }

  Array[String] filenames = fileName
}

task splitFile {
    File largeFile

    command {
        mkdir sandbox
        split -l 1 ${largeFile} sandbox/
    }

    output {
        Array[File] tiny_files = glob("sandbox/*")
    }
    runtime {
        docker: "ubuntu:latest"
    }
}

 Option 2

workflow use_file_of_filenames {
  File file_of_filenames
  call count_filenames_in_file { input: file_of_filenames = file_of_filenames }
  scatter (index in range(count_filenames_in_file.count)) {
    call operate_on_file { input: file_of_filenames = file_of_filenames, file_index = index }
  }
}

task count_filenames_in_file {
  File file_of_filenames
  command {
    wc -l < ${file_of_filenames}
  }
  output {
    Int count = read_int(stdout())
  }
}

task operate_on_file {
  File file_of_filenames
  Int file_index
  command {
    # 1: Get the appropriate file name from the list
    # 2: Operate on that file as a URL
  }
}

 

 

Alternatively, you can pass these files in as workflow inputs individually or collected in a tar.