python - How to see the progress bar of read_csv - Stack Overflow

PHOTO EMBED

Tue Dec 14 2021 02:02:54 GMT+0000 (UTC)

Saved by @weszerzad #python

def read_csv_pgbar(csv_path, chunksize, usecols, dtype=object):


    # print('Getting row count of csv file')

    rows = sum(1 for _ in open(csv_path, 'r')) - 1 # minus the header
    # chunks = rows//chunksize + 1
    # print('Reading csv file')
    chunk_list = []

    with tqdm(total=rows, desc='Rows read: ') as bar:
        for chunk in pd.read_csv(csv_path, chunksize=chunksize, usecols=usecols, dtype=dtype):
            chunk_list.append(chunk)
            bar.update(len(chunk))

    df = pd.concat((f for f in chunk_list), axis=0)
    print('Finish reading csv file')

    return df
content_copyCOPY

https://stackoverflow.com/questions/57174012/how-to-see-the-progress-bar-of-read-csv