def read_csv_pgbar(csv_path, chunksize, usecols, dtype=object):
    # print('Getting row count of csv file')
    rows = sum(1 for _ in open(csv_path, 'r')) - 1 # minus the header
    # chunks = rows//chunksize + 1
    # print('Reading csv file')
    chunk_list = []
    with tqdm(total=rows, desc='Rows read: ') as bar:
        for chunk in pd.read_csv(csv_path, chunksize=chunksize, usecols=usecols, dtype=dtype):
            chunk_list.append(chunk)
            bar.update(len(chunk))
    df = pd.concat((f for f in chunk_list), axis=0)
    print('Finish reading csv file')
    return df
                                 
                             
                        
Comments