def read_csv_pgbar(csv_path, chunksize, usecols, dtype=object):
# print('Getting row count of csv file')
rows = sum(1 for _ in open(csv_path, 'r')) - 1 # minus the header
# chunks = rows//chunksize + 1
# print('Reading csv file')
chunk_list = []
with tqdm(total=rows, desc='Rows read: ') as bar:
for chunk in pd.read_csv(csv_path, chunksize=chunksize, usecols=usecols, dtype=dtype):
chunk_list.append(chunk)
bar.update(len(chunk))
df = pd.concat((f for f in chunk_list), axis=0)
print('Finish reading csv file')
return df
Comments