# This reload library is just used for developing the REPUBLIC hOCR parser
# and can be removed once this module is stable.
%reload_ext autoreload
%autoreload 2
# This is needed to add the repo dir to the path so jupyter
# can load the modules in the scripts directory from the notebooks
import os
import sys
repo_dir = os.path.split(os.getcwd())[0]
print(repo_dir)
if repo_dir not in sys.path:
sys.path.append(repo_dir)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import json
import csv
data_dir = '../data/GoodReads'
books_10k_file = os.path.join(data_dir, 'goodreads_reviews-books_above_10k_reviews_lang.csv.gz')
reviewers_5k_file = os.path.join(data_dir, 'goodreads_reviews-reviewers_above_5k_reviews.csv.gz')
random_1M_file = os.path.join(data_dir, 'goodreads_reviews-random_sample_1M.csv.gz')
author_file = os.path.join(data_dir, 'goodreads_book_authors.csv.gz') # author information
book_file = os.path.join(data_dir, 'goodreads_books.csv.gz') # basic book metadata
/Users/marijnkoolen/Code/Huygens/scale
from scripts.text_tail_analysis import spacy_parse_store_from_dataframe
import spacy
import time
nlp = spacy.load('en_core_web_lg')
review_df = pd.read_csv(reviewers_5k_file, sep='\t', compression='gzip')
review_df = review_df[review_df.review_length > 0]
review_df = review_df[review_df.review_text.notna()]
fname = '../data/goodreads-reviews-reviewers_above_5k_reviews.doc_bin'
start = time.time()
spacy_parse_store_from_dataframe(fname, review_df, nlp)
print('took:', time.time() - start, 'seconds')
took: 2679.3893690109253 seconds
review_df = pd.read_csv(books_10k_file, sep='\t', compression='gzip')
review_df = review_df[review_df.review_length > 0]
review_df = review_df[review_df.review_text.notna()]
fname = '../data/goodreads-reviews-books_above_10k_reviews.doc_bin'
start = time.time()
spacy_parse_store_from_dataframe(fname, review_df, nlp)
print('took:', time.time() - start, 'seconds')
#review_df = pd.read_csv(random_1M_file, sep='\t', compression='gzip')
#review_df = review_df[review_df.review_length > 0]
#review_df = review_df[review_df.review_text.notna()]
fname = '../data/goodreads-reviews-random_sample_1M.doc_bin'
start = time.time()
spacy_parse_store_from_dataframe(fname, review_df, nlp)
print('took:', time.time() - start, 'seconds')
1000 reviews parsed in chunk 0 time: 18.446383237838745 2000 reviews parsed in chunk 0 time: 43.16318225860596 3000 reviews parsed in chunk 0 time: 62.45177602767944 4000 reviews parsed in chunk 0 time: 79.15793323516846 5000 reviews parsed in chunk 0 time: 97.15391635894775 6000 reviews parsed in chunk 0 time: 116.74516010284424 7000 reviews parsed in chunk 0 time: 141.39710116386414 8000 reviews parsed in chunk 0 time: 161.68016719818115 9000 reviews parsed in chunk 0 time: 181.23640823364258 10000 reviews parsed in chunk 0 time: 200.5376181602478 1000 reviews parsed in chunk 1 time: 222.11482119560242 2000 reviews parsed in chunk 1 time: 247.7168951034546 3000 reviews parsed in chunk 1 time: 269.7362439632416 4000 reviews parsed in chunk 1 time: 289.69021224975586 5000 reviews parsed in chunk 1 time: 307.922571182251 6000 reviews parsed in chunk 1 time: 335.9181761741638 7000 reviews parsed in chunk 1 time: 358.43035316467285 8000 reviews parsed in chunk 1 time: 379.01043915748596 9000 reviews parsed in chunk 1 time: 396.37506008148193 10000 reviews parsed in chunk 1 time: 421.48291206359863 1000 reviews parsed in chunk 2 time: 454.02014803886414 2000 reviews parsed in chunk 2 time: 473.8256483078003 3000 reviews parsed in chunk 2 time: 493.2523510456085 4000 reviews parsed in chunk 2 time: 516.1566581726074 5000 reviews parsed in chunk 2 time: 535.4996752738953 6000 reviews parsed in chunk 2 time: 556.4447753429413 7000 reviews parsed in chunk 2 time: 579.538516998291 8000 reviews parsed in chunk 2 time: 606.0051913261414 9000 reviews parsed in chunk 2 time: 624.0606572628021 10000 reviews parsed in chunk 2 time: 643.2514979839325 1000 reviews parsed in chunk 3 time: 667.895446062088 2000 reviews parsed in chunk 3 time: 692.5538692474365 3000 reviews parsed in chunk 3 time: 721.041980266571 4000 reviews parsed in chunk 3 time: 746.8004412651062 5000 reviews parsed in chunk 3 time: 766.4066162109375 6000 reviews parsed in chunk 3 time: 788.8778121471405 7000 reviews parsed in chunk 3 time: 806.1465392112732 8000 reviews parsed in chunk 3 time: 823.585921049118 9000 reviews parsed in chunk 3 time: 842.8740532398224 10000 reviews parsed in chunk 3 time: 864.3218281269073 1000 reviews parsed in chunk 4 time: 895.093424320221 2000 reviews parsed in chunk 4 time: 922.5533111095428 3000 reviews parsed in chunk 4 time: 940.9556171894073 4000 reviews parsed in chunk 4 time: 960.8183853626251 5000 reviews parsed in chunk 4 time: 981.2439041137695 6000 reviews parsed in chunk 4 time: 1002.0977690219879 7000 reviews parsed in chunk 4 time: 1022.1163110733032 8000 reviews parsed in chunk 4 time: 1043.9504861831665 9000 reviews parsed in chunk 4 time: 1062.4062910079956 10000 reviews parsed in chunk 4 time: 1084.6486983299255 1000 reviews parsed in chunk 5 time: 1110.832925081253 2000 reviews parsed in chunk 5 time: 1128.0733482837677 3000 reviews parsed in chunk 5 time: 1151.2114732265472 4000 reviews parsed in chunk 5 time: 1173.677267074585 5000 reviews parsed in chunk 5 time: 1196.767310142517 6000 reviews parsed in chunk 5 time: 1218.3948292732239 7000 reviews parsed in chunk 5 time: 1242.237641096115 8000 reviews parsed in chunk 5 time: 1262.6610372066498 9000 reviews parsed in chunk 5 time: 1281.7596590518951 10000 reviews parsed in chunk 5 time: 1315.3880620002747 1000 reviews parsed in chunk 6 time: 1341.234557390213 2000 reviews parsed in chunk 6 time: 1358.569864988327 3000 reviews parsed in chunk 6 time: 1377.6235520839691 4000 reviews parsed in chunk 6 time: 1397.6052820682526 5000 reviews parsed in chunk 6 time: 1419.0437293052673 6000 reviews parsed in chunk 6 time: 1437.043869972229 7000 reviews parsed in chunk 6 time: 1458.100298166275 8000 reviews parsed in chunk 6 time: 1478.7564132213593 9000 reviews parsed in chunk 6 time: 1503.6719951629639 10000 reviews parsed in chunk 6 time: 1526.7336521148682 1000 reviews parsed in chunk 7 time: 1548.6762671470642 2000 reviews parsed in chunk 7 time: 1570.3483572006226 3000 reviews parsed in chunk 7 time: 1589.6006262302399 4000 reviews parsed in chunk 7 time: 1608.7586271762848 5000 reviews parsed in chunk 7 time: 1627.4937331676483 6000 reviews parsed in chunk 7 time: 1647.3123352527618 7000 reviews parsed in chunk 7 time: 1666.2200241088867 8000 reviews parsed in chunk 7 time: 1688.5612082481384 9000 reviews parsed in chunk 7 time: 1703.9150893688202 10000 reviews parsed in chunk 7 time: 1720.8127381801605 1000 reviews parsed in chunk 8 time: 1739.2020680904388 2000 reviews parsed in chunk 8 time: 1758.841084241867 3000 reviews parsed in chunk 8 time: 1776.5872361660004 4000 reviews parsed in chunk 8 time: 1793.7250111103058 5000 reviews parsed in chunk 8 time: 1808.7906560897827 6000 reviews parsed in chunk 8 time: 1827.1630301475525 7000 reviews parsed in chunk 8 time: 1850.6321341991425 8000 reviews parsed in chunk 8 time: 1870.0721921920776 9000 reviews parsed in chunk 8 time: 1890.4879581928253 10000 reviews parsed in chunk 8 time: 1913.2566621303558 1000 reviews parsed in chunk 9 time: 1936.5263459682465 2000 reviews parsed in chunk 9 time: 1958.3888552188873 3000 reviews parsed in chunk 9 time: 1980.169357061386 4000 reviews parsed in chunk 9 time: 1999.7292201519012 5000 reviews parsed in chunk 9 time: 2017.2416951656342 6000 reviews parsed in chunk 9 time: 2039.8370382785797 7000 reviews parsed in chunk 9 time: 2062.0242371559143 8000 reviews parsed in chunk 9 time: 2084.724755048752 9000 reviews parsed in chunk 9 time: 2108.1422452926636 10000 reviews parsed in chunk 9 time: 2131.8849551677704 1000 reviews parsed in chunk 10 time: 2165.936471223831 2000 reviews parsed in chunk 10 time: 2191.2295863628387 3000 reviews parsed in chunk 10 time: 2215.705832004547 4000 reviews parsed in chunk 10 time: 2248.625798225403 5000 reviews parsed in chunk 10 time: 2269.8663642406464 6000 reviews parsed in chunk 10 time: 2289.7004492282867 7000 reviews parsed in chunk 10 time: 2312.146905183792 8000 reviews parsed in chunk 10 time: 2336.4080851078033 9000 reviews parsed in chunk 10 time: 2364.16405916214 10000 reviews parsed in chunk 10 time: 2391.113384962082 1000 reviews parsed in chunk 11 time: 2415.01802110672 2000 reviews parsed in chunk 11 time: 2436.508177280426 3000 reviews parsed in chunk 11 time: 2464.3569791316986 4000 reviews parsed in chunk 11 time: 2489.5179541110992 5000 reviews parsed in chunk 11 time: 2519.558494091034 6000 reviews parsed in chunk 11 time: 2541.5197081565857 7000 reviews parsed in chunk 11 time: 2562.340142250061 8000 reviews parsed in chunk 11 time: 2585.829861164093 9000 reviews parsed in chunk 11 time: 2616.434595108032 10000 reviews parsed in chunk 11 time: 2641.0534131526947 1000 reviews parsed in chunk 12 time: 2662.6365242004395 2000 reviews parsed in chunk 12 time: 2685.8587651252747 3000 reviews parsed in chunk 12 time: 2705.2730960845947 4000 reviews parsed in chunk 12 time: 2727.0888121128082 5000 reviews parsed in chunk 12 time: 2750.3439481258392 6000 reviews parsed in chunk 12 time: 2777.468977212906 7000 reviews parsed in chunk 12 time: 2799.7540471553802 8000 reviews parsed in chunk 12 time: 2818.138519048691 9000 reviews parsed in chunk 12 time: 2842.472331047058 10000 reviews parsed in chunk 12 time: 2867.6319229602814 1000 reviews parsed in chunk 13 time: 2889.6836490631104 2000 reviews parsed in chunk 13 time: 2908.63800907135 3000 reviews parsed in chunk 13 time: 2937.10009598732 4000 reviews parsed in chunk 13 time: 2962.5152661800385 5000 reviews parsed in chunk 13 time: 2980.9738051891327 6000 reviews parsed in chunk 13 time: 3003.220267057419 7000 reviews parsed in chunk 13 time: 3023.887515306473 8000 reviews parsed in chunk 13 time: 3045.0752472877502 9000 reviews parsed in chunk 13 time: 3064.3959732055664 10000 reviews parsed in chunk 13 time: 3083.581566095352 1000 reviews parsed in chunk 14 time: 3104.583559989929 2000 reviews parsed in chunk 14 time: 3123.835482120514 3000 reviews parsed in chunk 14 time: 3143.9641971588135 4000 reviews parsed in chunk 14 time: 3165.5368552207947 5000 reviews parsed in chunk 14 time: 3182.73481297493 6000 reviews parsed in chunk 14 time: 3207.995683193207 7000 reviews parsed in chunk 14 time: 3227.9646310806274 8000 reviews parsed in chunk 14 time: 3256.415479183197 9000 reviews parsed in chunk 14 time: 3276.8567731380463 10000 reviews parsed in chunk 14 time: 3293.9508872032166 1000 reviews parsed in chunk 15 time: 3315.396743297577 2000 reviews parsed in chunk 15 time: 3334.0287671089172 3000 reviews parsed in chunk 15 time: 3353.841185092926 4000 reviews parsed in chunk 15 time: 3370.718939304352 5000 reviews parsed in chunk 15 time: 3388.3400411605835 6000 reviews parsed in chunk 15 time: 3407.9381062984467 7000 reviews parsed in chunk 15 time: 3431.8810501098633 8000 reviews parsed in chunk 15 time: 3453.637490272522 9000 reviews parsed in chunk 15 time: 3472.813201189041 10000 reviews parsed in chunk 15 time: 3491.3623311519623 1000 reviews parsed in chunk 16 time: 3513.1947782039642 2000 reviews parsed in chunk 16 time: 3531.1344740390778 3000 reviews parsed in chunk 16 time: 3553.6973671913147 4000 reviews parsed in chunk 16 time: 3571.019867181778 5000 reviews parsed in chunk 16 time: 3591.28245306015 6000 reviews parsed in chunk 16 time: 3615.607928276062 7000 reviews parsed in chunk 16 time: 3629.8507091999054 8000 reviews parsed in chunk 16 time: 3652.9292101860046 9000 reviews parsed in chunk 16 time: 3671.3723261356354 10000 reviews parsed in chunk 16 time: 3691.7442910671234 1000 reviews parsed in chunk 17 time: 3713.549411058426 2000 reviews parsed in chunk 17 time: 3738.36731505394 3000 reviews parsed in chunk 17 time: 3756.5403180122375 4000 reviews parsed in chunk 17 time: 3777.542932033539 5000 reviews parsed in chunk 17 time: 3800.965085029602 6000 reviews parsed in chunk 17 time: 3825.8455653190613 7000 reviews parsed in chunk 17 time: 3846.508049249649 8000 reviews parsed in chunk 17 time: 3864.6262381076813 9000 reviews parsed in chunk 17 time: 3887.80105304718 10000 reviews parsed in chunk 17 time: 3904.725806236267 1000 reviews parsed in chunk 18 time: 3930.0132632255554 2000 reviews parsed in chunk 18 time: 3948.9682791233063 3000 reviews parsed in chunk 18 time: 3970.217052221298 4000 reviews parsed in chunk 18 time: 3989.6353871822357 5000 reviews parsed in chunk 18 time: 4009.880455970764 6000 reviews parsed in chunk 18 time: 4028.4486610889435 7000 reviews parsed in chunk 18 time: 4046.371663093567 8000 reviews parsed in chunk 18 time: 4068.041661262512 9000 reviews parsed in chunk 18 time: 4090.014288187027 10000 reviews parsed in chunk 18 time: 4109.227118253708 1000 reviews parsed in chunk 19 time: 4131.45760512352 2000 reviews parsed in chunk 19 time: 4152.4399580955505 3000 reviews parsed in chunk 19 time: 4171.616876125336 4000 reviews parsed in chunk 19 time: 4195.281736135483 5000 reviews parsed in chunk 19 time: 4223.555513143539 6000 reviews parsed in chunk 19 time: 4246.355656147003 7000 reviews parsed in chunk 19 time: 4264.385751247406 8000 reviews parsed in chunk 19 time: 4286.118881225586 9000 reviews parsed in chunk 19 time: 4308.053062200546 10000 reviews parsed in chunk 19 time: 4328.755905151367 1000 reviews parsed in chunk 20 time: 4362.160752296448 2000 reviews parsed in chunk 20 time: 4384.336077213287 3000 reviews parsed in chunk 20 time: 4405.2581651210785 4000 reviews parsed in chunk 20 time: 4422.305461168289 5000 reviews parsed in chunk 20 time: 4445.277947187424 6000 reviews parsed in chunk 20 time: 4468.495375156403 7000 reviews parsed in chunk 20 time: 4492.132717132568 8000 reviews parsed in chunk 20 time: 4509.980285167694 9000 reviews parsed in chunk 20 time: 4533.196135997772 10000 reviews parsed in chunk 20 time: 4556.943502187729 1000 reviews parsed in chunk 21 time: 4579.531319141388 2000 reviews parsed in chunk 21 time: 4600.668839216232 3000 reviews parsed in chunk 21 time: 4621.657905340195 4000 reviews parsed in chunk 21 time: 4642.813676118851 5000 reviews parsed in chunk 21 time: 4662.880634307861 6000 reviews parsed in chunk 21 time: 4686.138967037201 7000 reviews parsed in chunk 21 time: 4710.29412817955 8000 reviews parsed in chunk 21 time: 4735.6052412986755 9000 reviews parsed in chunk 21 time: 4760.839287042618 10000 reviews parsed in chunk 21 time: 4782.606028079987 1000 reviews parsed in chunk 22 time: 4806.255075216293 2000 reviews parsed in chunk 22 time: 4828.174202203751 3000 reviews parsed in chunk 22 time: 4845.1681871414185 4000 reviews parsed in chunk 22 time: 4871.571090936661 5000 reviews parsed in chunk 22 time: 4894.67534327507 6000 reviews parsed in chunk 22 time: 4920.7266302108765 7000 reviews parsed in chunk 22 time: 4938.901422262192 8000 reviews parsed in chunk 22 time: 4959.963854074478 9000 reviews parsed in chunk 22 time: 4981.424760103226 10000 reviews parsed in chunk 22 time: 5003.499162197113 1000 reviews parsed in chunk 23 time: 5027.176475286484 2000 reviews parsed in chunk 23 time: 5051.1266713142395 3000 reviews parsed in chunk 23 time: 5070.491139173508 4000 reviews parsed in chunk 23 time: 5092.673192024231 5000 reviews parsed in chunk 23 time: 5110.784128189087 6000 reviews parsed in chunk 23 time: 5131.167490243912 7000 reviews parsed in chunk 23 time: 5150.733951091766 8000 reviews parsed in chunk 23 time: 5175.515247106552 9000 reviews parsed in chunk 23 time: 5195.013009309769 10000 reviews parsed in chunk 23 time: 5213.1937782764435 1000 reviews parsed in chunk 24 time: 5237.975045204163 2000 reviews parsed in chunk 24 time: 5266.959020137787 3000 reviews parsed in chunk 24 time: 5291.478340148926 4000 reviews parsed in chunk 24 time: 5316.065537214279 5000 reviews parsed in chunk 24 time: 5335.70938706398 6000 reviews parsed in chunk 24 time: 5355.789422273636 7000 reviews parsed in chunk 24 time: 5377.425777196884 8000 reviews parsed in chunk 24 time: 5396.319365262985 9000 reviews parsed in chunk 24 time: 5420.64143705368 10000 reviews parsed in chunk 24 time: 5443.673362970352 1000 reviews parsed in chunk 25 time: 5472.166149139404 2000 reviews parsed in chunk 25 time: 5491.828714132309 3000 reviews parsed in chunk 25 time: 5509.9793882369995 4000 reviews parsed in chunk 25 time: 5529.5035400390625 5000 reviews parsed in chunk 25 time: 5552.456814289093 6000 reviews parsed in chunk 25 time: 5572.4864501953125 7000 reviews parsed in chunk 25 time: 5590.919986963272 8000 reviews parsed in chunk 25 time: 5609.460918188095 9000 reviews parsed in chunk 25 time: 5630.176309108734 10000 reviews parsed in chunk 25 time: 5657.0486171245575 1000 reviews parsed in chunk 26 time: 5679.264386177063 2000 reviews parsed in chunk 26 time: 5696.405147314072 3000 reviews parsed in chunk 26 time: 5716.467316150665 4000 reviews parsed in chunk 26 time: 5738.663419246674 5000 reviews parsed in chunk 26 time: 5763.491099119186 6000 reviews parsed in chunk 26 time: 5781.174111127853 7000 reviews parsed in chunk 26 time: 5799.649218082428 8000 reviews parsed in chunk 26 time: 5819.371384143829 9000 reviews parsed in chunk 26 time: 5838.11819601059 10000 reviews parsed in chunk 26 time: 5857.876441240311 1000 reviews parsed in chunk 27 time: 5882.033170938492 2000 reviews parsed in chunk 27 time: 5906.720479011536 3000 reviews parsed in chunk 27 time: 5927.577264308929 4000 reviews parsed in chunk 27 time: 5947.984906196594 5000 reviews parsed in chunk 27 time: 5972.269669294357 6000 reviews parsed in chunk 27 time: 5992.031666994095 7000 reviews parsed in chunk 27 time: 6015.929742097855 8000 reviews parsed in chunk 27 time: 6038.866021156311 9000 reviews parsed in chunk 27 time: 6059.146010160446 10000 reviews parsed in chunk 27 time: 6081.820103168488 1000 reviews parsed in chunk 28 time: 6106.947947025299 2000 reviews parsed in chunk 28 time: 6125.812674045563 3000 reviews parsed in chunk 28 time: 6148.866105079651 4000 reviews parsed in chunk 28 time: 6168.4242742061615 5000 reviews parsed in chunk 28 time: 6188.432215213776 6000 reviews parsed in chunk 28 time: 6209.139053106308 7000 reviews parsed in chunk 28 time: 6229.793969154358 8000 reviews parsed in chunk 28 time: 6256.348958015442 9000 reviews parsed in chunk 28 time: 6277.676685094833 10000 reviews parsed in chunk 28 time: 6297.396054029465 1000 reviews parsed in chunk 29 time: 6325.4676151275635 2000 reviews parsed in chunk 29 time: 6350.877683162689 3000 reviews parsed in chunk 29 time: 6374.037668228149 4000 reviews parsed in chunk 29 time: 6404.932452201843 5000 reviews parsed in chunk 29 time: 6427.232709169388 6000 reviews parsed in chunk 29 time: 6457.245125293732 7000 reviews parsed in chunk 29 time: 6485.711646080017 8000 reviews parsed in chunk 29 time: 6513.620273351669 9000 reviews parsed in chunk 29 time: 6544.594169139862 10000 reviews parsed in chunk 29 time: 6566.061857223511 1000 reviews parsed in chunk 30 time: 6598.77446603775 2000 reviews parsed in chunk 30 time: 6623.27952003479 3000 reviews parsed in chunk 30 time: 6642.721805095673 4000 reviews parsed in chunk 30 time: 6672.172251224518 5000 reviews parsed in chunk 30 time: 6697.679239988327 6000 reviews parsed in chunk 30 time: 6714.669230222702 7000 reviews parsed in chunk 30 time: 6735.540992021561 8000 reviews parsed in chunk 30 time: 6758.185130119324 9000 reviews parsed in chunk 30 time: 6780.815145969391 10000 reviews parsed in chunk 30 time: 6799.6361973285675 1000 reviews parsed in chunk 31 time: 6824.963599205017 2000 reviews parsed in chunk 31 time: 6847.149702310562 3000 reviews parsed in chunk 31 time: 6865.85693526268 4000 reviews parsed in chunk 31 time: 6891.198043346405 5000 reviews parsed in chunk 31 time: 6918.17499423027 6000 reviews parsed in chunk 31 time: 6938.401337146759 7000 reviews parsed in chunk 31 time: 6959.651329040527 8000 reviews parsed in chunk 31 time: 6979.160242319107 9000 reviews parsed in chunk 31 time: 7000.356417179108 10000 reviews parsed in chunk 31 time: 7018.657244920731 1000 reviews parsed in chunk 32 time: 7041.422184944153 2000 reviews parsed in chunk 32 time: 7063.660144329071 3000 reviews parsed in chunk 32 time: 7089.644077062607 4000 reviews parsed in chunk 32 time: 7111.205897092819 5000 reviews parsed in chunk 32 time: 7129.748033046722 6000 reviews parsed in chunk 32 time: 7152.333125114441 7000 reviews parsed in chunk 32 time: 7172.004853010178 8000 reviews parsed in chunk 32 time: 7191.699996232986 9000 reviews parsed in chunk 32 time: 7213.057741165161 10000 reviews parsed in chunk 32 time: 7232.806185007095 1000 reviews parsed in chunk 33 time: 7253.930625200272 2000 reviews parsed in chunk 33 time: 7276.528643131256 3000 reviews parsed in chunk 33 time: 7298.397517204285 4000 reviews parsed in chunk 33 time: 7318.528701305389 5000 reviews parsed in chunk 33 time: 7346.902320146561 6000 reviews parsed in chunk 33 time: 7364.700313091278 7000 reviews parsed in chunk 33 time: 7384.342645168304 8000 reviews parsed in chunk 33 time: 7412.276249170303 9000 reviews parsed in chunk 33 time: 7431.202918291092 10000 reviews parsed in chunk 33 time: 7461.092485189438 1000 reviews parsed in chunk 34 time: 7492.617742300034 2000 reviews parsed in chunk 34 time: 7513.081768989563 3000 reviews parsed in chunk 34 time: 7535.805594205856 4000 reviews parsed in chunk 34 time: 7555.35343003273 5000 reviews parsed in chunk 34 time: 7582.819936037064 6000 reviews parsed in chunk 34 time: 7605.834584236145 7000 reviews parsed in chunk 34 time: 7625.608340978622 8000 reviews parsed in chunk 34 time: 7645.009799957275 9000 reviews parsed in chunk 34 time: 7671.575476169586 10000 reviews parsed in chunk 34 time: 7700.104557275772 1000 reviews parsed in chunk 35 time: 7726.657277345657 2000 reviews parsed in chunk 35 time: 7751.549492120743 3000 reviews parsed in chunk 35 time: 7776.197065114975 4000 reviews parsed in chunk 35 time: 7805.319934129715 5000 reviews parsed in chunk 35 time: 7835.551687955856 6000 reviews parsed in chunk 35 time: 7876.355846166611 7000 reviews parsed in chunk 35 time: 7910.400229215622 8000 reviews parsed in chunk 35 time: 7944.861142158508 9000 reviews parsed in chunk 35 time: 7969.740108251572 10000 reviews parsed in chunk 35 time: 7999.028450250626 1000 reviews parsed in chunk 36 time: 8025.876429080963 2000 reviews parsed in chunk 36 time: 8055.783996343613 3000 reviews parsed in chunk 36 time: 8076.388412237167 4000 reviews parsed in chunk 36 time: 8096.389607191086 5000 reviews parsed in chunk 36 time: 8122.4752950668335 6000 reviews parsed in chunk 36 time: 8148.926705360413 7000 reviews parsed in chunk 36 time: 8168.692263364792 8000 reviews parsed in chunk 36 time: 8193.569808244705 9000 reviews parsed in chunk 36 time: 8221.808492183685 10000 reviews parsed in chunk 36 time: 8248.227114200592 1000 reviews parsed in chunk 37 time: 8270.902371168137 2000 reviews parsed in chunk 37 time: 8293.229907989502 3000 reviews parsed in chunk 37 time: 8323.759462356567 4000 reviews parsed in chunk 37 time: 8350.643237113953 5000 reviews parsed in chunk 37 time: 8368.55176615715 6000 reviews parsed in chunk 37 time: 8389.850090026855 7000 reviews parsed in chunk 37 time: 8416.286515951157 8000 reviews parsed in chunk 37 time: 8435.78335738182 9000 reviews parsed in chunk 37 time: 8457.12298130989 10000 reviews parsed in chunk 37 time: 8481.452584028244 1000 reviews parsed in chunk 38 time: 8507.437704086304 2000 reviews parsed in chunk 38 time: 8529.8208091259 3000 reviews parsed in chunk 38 time: 8555.755328178406 4000 reviews parsed in chunk 38 time: 8577.313930034637 5000 reviews parsed in chunk 38 time: 8600.917830228806 6000 reviews parsed in chunk 38 time: 8621.844758033752 7000 reviews parsed in chunk 38 time: 8646.773069143295 8000 reviews parsed in chunk 38 time: 8669.567224264145 9000 reviews parsed in chunk 38 time: 8690.38021826744 10000 reviews parsed in chunk 38 time: 8715.916156291962 1000 reviews parsed in chunk 39 time: 8743.336080312729 2000 reviews parsed in chunk 39 time: 8764.155838012695 3000 reviews parsed in chunk 39 time: 8784.854631185532 4000 reviews parsed in chunk 39 time: 8827.332693099976 5000 reviews parsed in chunk 39 time: 8850.71792626381 6000 reviews parsed in chunk 39 time: 8880.510625123978 7000 reviews parsed in chunk 39 time: 8900.772520065308 8000 reviews parsed in chunk 39 time: 8921.170827150345 9000 reviews parsed in chunk 39 time: 8943.326516151428 10000 reviews parsed in chunk 39 time: 8972.971194267273 1000 reviews parsed in chunk 40 time: 8998.474375963211 2000 reviews parsed in chunk 40 time: 9020.8356051445 3000 reviews parsed in chunk 40 time: 9047.161276340485 4000 reviews parsed in chunk 40 time: 9068.559699058533 5000 reviews parsed in chunk 40 time: 9097.467314958572 6000 reviews parsed in chunk 40 time: 9126.69700217247 7000 reviews parsed in chunk 40 time: 9145.495970249176 8000 reviews parsed in chunk 40 time: 9164.514186143875 9000 reviews parsed in chunk 40 time: 9185.158508062363 10000 reviews parsed in chunk 40 time: 9202.940984249115 1000 reviews parsed in chunk 41 time: 9229.003517389297 2000 reviews parsed in chunk 41 time: 9254.784424066544 3000 reviews parsed in chunk 41 time: 9275.767100334167 4000 reviews parsed in chunk 41 time: 9293.673767328262 5000 reviews parsed in chunk 41 time: 9313.238007068634 6000 reviews parsed in chunk 41 time: 9332.183881044388 7000 reviews parsed in chunk 41 time: 9354.199260950089 8000 reviews parsed in chunk 41 time: 9375.658994197845 9000 reviews parsed in chunk 41 time: 9400.827795028687 10000 reviews parsed in chunk 41 time: 9419.685651063919 1000 reviews parsed in chunk 42 time: 9442.106191158295 2000 reviews parsed in chunk 42 time: 9467.056703090668 3000 reviews parsed in chunk 42 time: 9486.460868120193 4000 reviews parsed in chunk 42 time: 9504.820854187012 5000 reviews parsed in chunk 42 time: 9525.49921822548 6000 reviews parsed in chunk 42 time: 9547.892915964127 7000 reviews parsed in chunk 42 time: 9571.187841176987 8000 reviews parsed in chunk 42 time: 9598.095810174942 9000 reviews parsed in chunk 42 time: 9619.964022159576 10000 reviews parsed in chunk 42 time: 9637.857434272766 1000 reviews parsed in chunk 43 time: 9661.893481016159 2000 reviews parsed in chunk 43 time: 9681.875007152557 3000 reviews parsed in chunk 43 time: 9708.12966132164 4000 reviews parsed in chunk 43 time: 9735.25321507454 5000 reviews parsed in chunk 43 time: 9757.3204600811 6000 reviews parsed in chunk 43 time: 9777.998239994049 7000 reviews parsed in chunk 43 time: 9802.622776269913 8000 reviews parsed in chunk 43 time: 9831.084393024445 9000 reviews parsed in chunk 43 time: 9853.224723100662 10000 reviews parsed in chunk 43 time: 9879.801295280457 1000 reviews parsed in chunk 44 time: 9901.843297243118 2000 reviews parsed in chunk 44 time: 9924.269957304 3000 reviews parsed in chunk 44 time: 9955.500249147415 4000 reviews parsed in chunk 44 time: 9980.440467119217 5000 reviews parsed in chunk 44 time: 10007.676764965057 6000 reviews parsed in chunk 44 time: 10028.425697088242 7000 reviews parsed in chunk 44 time: 10056.842160224915 8000 reviews parsed in chunk 44 time: 10086.527171134949 9000 reviews parsed in chunk 44 time: 10106.712465286255 10000 reviews parsed in chunk 44 time: 10135.88037610054 1000 reviews parsed in chunk 45 time: 10161.655883073807 2000 reviews parsed in chunk 45 time: 10181.433017253876 3000 reviews parsed in chunk 45 time: 10202.20443701744 4000 reviews parsed in chunk 45 time: 10222.121057033539 5000 reviews parsed in chunk 45 time: 10243.451151132584 6000 reviews parsed in chunk 45 time: 10262.203502178192 7000 reviews parsed in chunk 45 time: 10295.180330991745 8000 reviews parsed in chunk 45 time: 10315.40245103836 9000 reviews parsed in chunk 45 time: 10339.333052158356 10000 reviews parsed in chunk 45 time: 10361.63627910614 1000 reviews parsed in chunk 46 time: 10384.896146297455 2000 reviews parsed in chunk 46 time: 10407.221068143845 3000 reviews parsed in chunk 46 time: 10431.850106239319 4000 reviews parsed in chunk 46 time: 10457.125293254852 5000 reviews parsed in chunk 46 time: 10476.626095294952 6000 reviews parsed in chunk 46 time: 10494.885470151901 7000 reviews parsed in chunk 46 time: 10517.301290035248 8000 reviews parsed in chunk 46 time: 10539.021446228027 9000 reviews parsed in chunk 46 time: 10561.831044197083 10000 reviews parsed in chunk 46 time: 10584.782248020172 1000 reviews parsed in chunk 47 time: 10605.205311059952 2000 reviews parsed in chunk 47 time: 10625.083648204803 3000 reviews parsed in chunk 47 time: 10646.745663166046 4000 reviews parsed in chunk 47 time: 10665.800614356995 5000 reviews parsed in chunk 47 time: 10688.970434188843 6000 reviews parsed in chunk 47 time: 10709.150475025177 7000 reviews parsed in chunk 47 time: 10732.934199094772 8000 reviews parsed in chunk 47 time: 10751.823265075684 9000 reviews parsed in chunk 47 time: 10771.483110189438 10000 reviews parsed in chunk 47 time: 10790.239776134491 1000 reviews parsed in chunk 48 time: 10812.158676147461 2000 reviews parsed in chunk 48 time: 10831.34462428093 3000 reviews parsed in chunk 48 time: 10855.03437423706 4000 reviews parsed in chunk 48 time: 10885.604751110077 5000 reviews parsed in chunk 48 time: 10908.53788614273 6000 reviews parsed in chunk 48 time: 10933.551929235458 7000 reviews parsed in chunk 48 time: 10956.194253206253 8000 reviews parsed in chunk 48 time: 10977.130536079407 9000 reviews parsed in chunk 48 time: 10999.185787200928 10000 reviews parsed in chunk 48 time: 11018.8436191082 1000 reviews parsed in chunk 49 time: 11044.663279294968 2000 reviews parsed in chunk 49 time: 11062.60732126236 3000 reviews parsed in chunk 49 time: 11084.754827022552 4000 reviews parsed in chunk 49 time: 11103.359198093414 5000 reviews parsed in chunk 49 time: 11122.671314954758 6000 reviews parsed in chunk 49 time: 11144.42544412613 7000 reviews parsed in chunk 49 time: 11162.316866159439 8000 reviews parsed in chunk 49 time: 11182.126830101013 9000 reviews parsed in chunk 49 time: 11199.33187508583 10000 reviews parsed in chunk 49 time: 11216.750994205475 1000 reviews parsed in chunk 50 time: 11238.961301088333 2000 reviews parsed in chunk 50 time: 11257.91182422638 3000 reviews parsed in chunk 50 time: 11279.480054140091 4000 reviews parsed in chunk 50 time: 11297.668485164642 5000 reviews parsed in chunk 50 time: 11318.530770301819 6000 reviews parsed in chunk 50 time: 11338.02347612381 7000 reviews parsed in chunk 50 time: 11357.929661273956 8000 reviews parsed in chunk 50 time: 11375.452706098557 9000 reviews parsed in chunk 50 time: 11403.957764148712 10000 reviews parsed in chunk 50 time: 11423.557661056519 1000 reviews parsed in chunk 51 time: 11442.901893138885 2000 reviews parsed in chunk 51 time: 11461.761981964111 3000 reviews parsed in chunk 51 time: 11481.279313325882 4000 reviews parsed in chunk 51 time: 11501.723847150803 5000 reviews parsed in chunk 51 time: 11519.182747125626 6000 reviews parsed in chunk 51 time: 11537.404419183731 7000 reviews parsed in chunk 51 time: 11558.128064155579 8000 reviews parsed in chunk 51 time: 11575.65494298935 9000 reviews parsed in chunk 51 time: 11593.804460048676 10000 reviews parsed in chunk 51 time: 11613.192924976349 1000 reviews parsed in chunk 52 time: 11636.410618305206 2000 reviews parsed in chunk 52 time: 11656.907097101212 3000 reviews parsed in chunk 52 time: 11676.736782073975 4000 reviews parsed in chunk 52 time: 11698.282716989517 5000 reviews parsed in chunk 52 time: 11720.028125047684 6000 reviews parsed in chunk 52 time: 11742.858539104462 7000 reviews parsed in chunk 52 time: 11760.39478111267 8000 reviews parsed in chunk 52 time: 11786.357823133469 9000 reviews parsed in chunk 52 time: 11807.025710105896 10000 reviews parsed in chunk 52 time: 11830.055446147919 1000 reviews parsed in chunk 53 time: 11853.037091255188 2000 reviews parsed in chunk 53 time: 11871.501853942871 3000 reviews parsed in chunk 53 time: 11888.84639430046 4000 reviews parsed in chunk 53 time: 11904.987940311432 5000 reviews parsed in chunk 53 time: 11933.506968259811 6000 reviews parsed in chunk 53 time: 11953.103229999542 7000 reviews parsed in chunk 53 time: 11971.817768096924 8000 reviews parsed in chunk 53 time: 11991.800091266632 9000 reviews parsed in chunk 53 time: 12009.859810113907 10000 reviews parsed in chunk 53 time: 12028.625717163086 1000 reviews parsed in chunk 54 time: 12056.094970226288 2000 reviews parsed in chunk 54 time: 12081.445216178894 3000 reviews parsed in chunk 54 time: 12100.881581068039 4000 reviews parsed in chunk 54 time: 12124.796130180359 5000 reviews parsed in chunk 54 time: 12146.515869140625 6000 reviews parsed in chunk 54 time: 12162.228976249695 7000 reviews parsed in chunk 54 time: 12184.7745013237 8000 reviews parsed in chunk 54 time: 12201.060132265091 9000 reviews parsed in chunk 54 time: 12219.087420225143 10000 reviews parsed in chunk 54 time: 12236.324463129044 1000 reviews parsed in chunk 55 time: 12258.26071023941 2000 reviews parsed in chunk 55 time: 12277.092606067657 3000 reviews parsed in chunk 55 time: 12300.080834150314 4000 reviews parsed in chunk 55 time: 12319.227262973785 5000 reviews parsed in chunk 55 time: 12337.546114206314 6000 reviews parsed in chunk 55 time: 12357.7039103508 7000 reviews parsed in chunk 55 time: 12376.63864517212 8000 reviews parsed in chunk 55 time: 12403.008357286453 9000 reviews parsed in chunk 55 time: 12419.832560300827 10000 reviews parsed in chunk 55 time: 12442.162538051605 1000 reviews parsed in chunk 56 time: 12467.63686823845 2000 reviews parsed in chunk 56 time: 12483.689739227295 3000 reviews parsed in chunk 56 time: 12500.239958047867 4000 reviews parsed in chunk 56 time: 12519.267899274826 5000 reviews parsed in chunk 56 time: 12541.05047917366 6000 reviews parsed in chunk 56 time: 12560.71294426918 7000 reviews parsed in chunk 56 time: 12580.614422082901 8000 reviews parsed in chunk 56 time: 12601.582868099213 9000 reviews parsed in chunk 56 time: 12632.282105207443 10000 reviews parsed in chunk 56 time: 12651.092165231705 1000 reviews parsed in chunk 57 time: 12675.301188230515 2000 reviews parsed in chunk 57 time: 12693.738813400269 3000 reviews parsed in chunk 57 time: 12710.587569236755 4000 reviews parsed in chunk 57 time: 12726.110520124435 5000 reviews parsed in chunk 57 time: 12747.0886490345 6000 reviews parsed in chunk 57 time: 12764.743722200394 7000 reviews parsed in chunk 57 time: 12786.599560022354 8000 reviews parsed in chunk 57 time: 12806.988666296005 9000 reviews parsed in chunk 57 time: 12826.205911159515 10000 reviews parsed in chunk 57 time: 12845.109956264496 1000 reviews parsed in chunk 58 time: 12863.683940172195 2000 reviews parsed in chunk 58 time: 12891.277605295181 3000 reviews parsed in chunk 58 time: 12909.653385162354 4000 reviews parsed in chunk 58 time: 12929.386226177216 5000 reviews parsed in chunk 58 time: 12949.120481014252 6000 reviews parsed in chunk 58 time: 12969.300856113434 7000 reviews parsed in chunk 58 time: 12988.494646072388 8000 reviews parsed in chunk 58 time: 13015.092257022858 9000 reviews parsed in chunk 58 time: 13038.077720165253 10000 reviews parsed in chunk 58 time: 13056.474444150925 1000 reviews parsed in chunk 59 time: 13083.81039428711 2000 reviews parsed in chunk 59 time: 13107.169988155365 3000 reviews parsed in chunk 59 time: 13132.161717176437 4000 reviews parsed in chunk 59 time: 13153.954581022263 5000 reviews parsed in chunk 59 time: 13175.954944133759 6000 reviews parsed in chunk 59 time: 13197.980729103088 7000 reviews parsed in chunk 59 time: 13218.682133197784 8000 reviews parsed in chunk 59 time: 13237.452928066254 9000 reviews parsed in chunk 59 time: 13256.13511133194 10000 reviews parsed in chunk 59 time: 13277.639600276947 1000 reviews parsed in chunk 60 time: 13300.804004192352 2000 reviews parsed in chunk 60 time: 13319.472497224808 3000 reviews parsed in chunk 60 time: 13344.355979204178 4000 reviews parsed in chunk 60 time: 13363.817995071411 5000 reviews parsed in chunk 60 time: 13383.073131084442 6000 reviews parsed in chunk 60 time: 13408.549631118774 7000 reviews parsed in chunk 60 time: 13438.707126140594 8000 reviews parsed in chunk 60 time: 13459.091896057129 9000 reviews parsed in chunk 60 time: 13479.410695314407 10000 reviews parsed in chunk 60 time: 13498.691390991211 1000 reviews parsed in chunk 61 time: 13526.592142105103 2000 reviews parsed in chunk 61 time: 13550.136333227158 3000 reviews parsed in chunk 61 time: 13569.24829530716 4000 reviews parsed in chunk 61 time: 13586.770002365112 5000 reviews parsed in chunk 61 time: 13603.998241186142 6000 reviews parsed in chunk 61 time: 13625.388283252716 7000 reviews parsed in chunk 61 time: 13650.020727157593 8000 reviews parsed in chunk 61 time: 13671.232648134232 9000 reviews parsed in chunk 61 time: 13695.32043504715 10000 reviews parsed in chunk 61 time: 13717.553586244583 1000 reviews parsed in chunk 62 time: 13735.760975122452 2000 reviews parsed in chunk 62 time: 13754.31618309021 3000 reviews parsed in chunk 62 time: 13776.874433040619 4000 reviews parsed in chunk 62 time: 13798.316949129105 5000 reviews parsed in chunk 62 time: 13829.397317171097 6000 reviews parsed in chunk 62 time: 13843.501347064972 7000 reviews parsed in chunk 62 time: 13848.31553530693 8000 reviews parsed in chunk 62 time: 13874.08555817604 9000 reviews parsed in chunk 62 time: 13893.875964164734 10000 reviews parsed in chunk 62 time: 13913.093252182007 1000 reviews parsed in chunk 63 time: 13934.106302022934 2000 reviews parsed in chunk 63 time: 13952.18447637558 3000 reviews parsed in chunk 63 time: 13971.945965051651 4000 reviews parsed in chunk 63 time: 13990.128107070923 5000 reviews parsed in chunk 63 time: 14010.06513428688 6000 reviews parsed in chunk 63 time: 14030.573921203613 7000 reviews parsed in chunk 63 time: 14056.114118337631 8000 reviews parsed in chunk 63 time: 14078.635420322418 9000 reviews parsed in chunk 63 time: 14099.892295360565 10000 reviews parsed in chunk 63 time: 14118.708259105682 1000 reviews parsed in chunk 64 time: 14140.472860336304 2000 reviews parsed in chunk 64 time: 14158.542304039001 3000 reviews parsed in chunk 64 time: 14178.092107057571 4000 reviews parsed in chunk 64 time: 14196.498616218567 5000 reviews parsed in chunk 64 time: 14219.39281821251 6000 reviews parsed in chunk 64 time: 14236.871826171875 7000 reviews parsed in chunk 64 time: 14255.815813302994 8000 reviews parsed in chunk 64 time: 14276.186637163162 9000 reviews parsed in chunk 64 time: 14293.578242063522 10000 reviews parsed in chunk 64 time: 14313.101652145386 1000 reviews parsed in chunk 65 time: 14333.795100927353 2000 reviews parsed in chunk 65 time: 14353.082555294037 3000 reviews parsed in chunk 65 time: 14374.238893270493 4000 reviews parsed in chunk 65 time: 14395.636813163757 5000 reviews parsed in chunk 65 time: 14416.666556358337 6000 reviews parsed in chunk 65 time: 14436.649851322174 7000 reviews parsed in chunk 65 time: 14452.787636995316 8000 reviews parsed in chunk 65 time: 14474.95685505867 9000 reviews parsed in chunk 65 time: 14491.412346124649 10000 reviews parsed in chunk 65 time: 14513.141211271286 1000 reviews parsed in chunk 66 time: 14537.668934345245 2000 reviews parsed in chunk 66 time: 14557.048445224762 3000 reviews parsed in chunk 66 time: 14574.319853067398 4000 reviews parsed in chunk 66 time: 14593.453294277191 5000 reviews parsed in chunk 66 time: 14615.675559282303 6000 reviews parsed in chunk 66 time: 14636.74473810196 7000 reviews parsed in chunk 66 time: 14655.053217172623 8000 reviews parsed in chunk 66 time: 14672.511563062668 9000 reviews parsed in chunk 66 time: 14694.75093626976 10000 reviews parsed in chunk 66 time: 14714.470449209213 1000 reviews parsed in chunk 67 time: 14740.615734100342 2000 reviews parsed in chunk 67 time: 14760.883912086487 3000 reviews parsed in chunk 67 time: 14782.496252298355 4000 reviews parsed in chunk 67 time: 14801.209240198135 5000 reviews parsed in chunk 67 time: 14820.384624242783 6000 reviews parsed in chunk 67 time: 14845.758279323578 7000 reviews parsed in chunk 67 time: 14867.606932163239 8000 reviews parsed in chunk 67 time: 14885.971633195877 9000 reviews parsed in chunk 67 time: 14914.83126616478 10000 reviews parsed in chunk 67 time: 14934.768556118011 1000 reviews parsed in chunk 68 time: 14960.082795143127 2000 reviews parsed in chunk 68 time: 14978.881568193436 3000 reviews parsed in chunk 68 time: 14997.585072278976 4000 reviews parsed in chunk 68 time: 15015.643198013306 5000 reviews parsed in chunk 68 time: 15036.747641086578 6000 reviews parsed in chunk 68 time: 15062.5241522789 7000 reviews parsed in chunk 68 time: 15082.232488155365 8000 reviews parsed in chunk 68 time: 15101.353335142136 9000 reviews parsed in chunk 68 time: 15122.913809299469 10000 reviews parsed in chunk 68 time: 15143.754317045212 1000 reviews parsed in chunk 69 time: 15163.30172920227 2000 reviews parsed in chunk 69 time: 15183.786132097244 3000 reviews parsed in chunk 69 time: 15198.760411262512 4000 reviews parsed in chunk 69 time: 15219.921910047531 5000 reviews parsed in chunk 69 time: 15237.734851121902 6000 reviews parsed in chunk 69 time: 15261.92279624939 7000 reviews parsed in chunk 69 time: 15281.235874176025 8000 reviews parsed in chunk 69 time: 15303.91552901268 9000 reviews parsed in chunk 69 time: 15319.975182294846 10000 reviews parsed in chunk 69 time: 15340.920540332794 1000 reviews parsed in chunk 70 time: 15365.367648124695 2000 reviews parsed in chunk 70 time: 15382.0005671978 3000 reviews parsed in chunk 70 time: 15400.174500226974 4000 reviews parsed in chunk 70 time: 15419.726937294006 5000 reviews parsed in chunk 70 time: 15439.583503246307 6000 reviews parsed in chunk 70 time: 15456.705336093903 7000 reviews parsed in chunk 70 time: 15479.497033119202 8000 reviews parsed in chunk 70 time: 15501.924315214157 9000 reviews parsed in chunk 70 time: 15518.180546045303 10000 reviews parsed in chunk 70 time: 15535.838989257812 1000 reviews parsed in chunk 71 time: 15557.891710042953 2000 reviews parsed in chunk 71 time: 15579.717023134232 3000 reviews parsed in chunk 71 time: 15601.56843328476 4000 reviews parsed in chunk 71 time: 15619.979279279709 5000 reviews parsed in chunk 71 time: 15640.35290813446 6000 reviews parsed in chunk 71 time: 15661.622069120407 7000 reviews parsed in chunk 71 time: 15677.898786067963 8000 reviews parsed in chunk 71 time: 15697.519268274307 9000 reviews parsed in chunk 71 time: 15719.186326026917 10000 reviews parsed in chunk 71 time: 15745.253078222275 1000 reviews parsed in chunk 72 time: 15767.399358987808 2000 reviews parsed in chunk 72 time: 15786.735873222351 3000 reviews parsed in chunk 72 time: 15806.883726119995 4000 reviews parsed in chunk 72 time: 15826.617906332016 5000 reviews parsed in chunk 72 time: 15844.291676998138 6000 reviews parsed in chunk 72 time: 15865.536829948425 7000 reviews parsed in chunk 72 time: 15884.68942117691 8000 reviews parsed in chunk 72 time: 15904.814008235931 9000 reviews parsed in chunk 72 time: 15924.899453401566 10000 reviews parsed in chunk 72 time: 15943.37010717392 1000 reviews parsed in chunk 73 time: 15965.476481199265 2000 reviews parsed in chunk 73 time: 15982.546632051468 3000 reviews parsed in chunk 73 time: 16001.819144964218 4000 reviews parsed in chunk 73 time: 16024.1466319561 5000 reviews parsed in chunk 73 time: 16041.86812210083 6000 reviews parsed in chunk 73 time: 16062.128376245499 7000 reviews parsed in chunk 73 time: 16079.596027135849 8000 reviews parsed in chunk 73 time: 16100.356583356857 9000 reviews parsed in chunk 73 time: 16117.328288078308 10000 reviews parsed in chunk 73 time: 16137.955673217773 1000 reviews parsed in chunk 74 time: 16160.392466306686 2000 reviews parsed in chunk 74 time: 16184.886550188065 3000 reviews parsed in chunk 74 time: 16203.115820169449 4000 reviews parsed in chunk 74 time: 16220.48317694664 5000 reviews parsed in chunk 74 time: 16237.317954063416 6000 reviews parsed in chunk 74 time: 16266.764668226242 7000 reviews parsed in chunk 74 time: 16285.188127994537 8000 reviews parsed in chunk 74 time: 16305.164227247238 9000 reviews parsed in chunk 74 time: 16325.263085126877 10000 reviews parsed in chunk 74 time: 16346.849565029144 1000 reviews parsed in chunk 75 time: 16369.073479175568 2000 reviews parsed in chunk 75 time: 16391.62385416031 3000 reviews parsed in chunk 75 time: 16409.08402109146 4000 reviews parsed in chunk 75 time: 16432.869274139404 5000 reviews parsed in chunk 75 time: 16451.840345144272 6000 reviews parsed in chunk 75 time: 16472.65037918091 7000 reviews parsed in chunk 75 time: 16491.923777103424 8000 reviews parsed in chunk 75 time: 16513.363913059235 9000 reviews parsed in chunk 75 time: 16536.080496311188 10000 reviews parsed in chunk 75 time: 16554.474813222885 1000 reviews parsed in chunk 76 time: 16573.82795214653 2000 reviews parsed in chunk 76 time: 16591.291813135147 3000 reviews parsed in chunk 76 time: 16615.636520147324 4000 reviews parsed in chunk 76 time: 16633.657586336136 5000 reviews parsed in chunk 76 time: 16649.51060628891 6000 reviews parsed in chunk 76 time: 16668.349790096283 7000 reviews parsed in chunk 76 time: 16689.63602423668 8000 reviews parsed in chunk 76 time: 16712.46386218071 9000 reviews parsed in chunk 76 time: 16729.295116186142 10000 reviews parsed in chunk 76 time: 16750.835655212402 1000 reviews parsed in chunk 77 time: 16770.595815181732 2000 reviews parsed in chunk 77 time: 16787.295783281326 3000 reviews parsed in chunk 77 time: 16806.12268614769 4000 reviews parsed in chunk 77 time: 16825.083057165146 5000 reviews parsed in chunk 77 time: 16843.189648151398 6000 reviews parsed in chunk 77 time: 16866.2554602623 7000 reviews parsed in chunk 77 time: 16884.396163225174 8000 reviews parsed in chunk 77 time: 16904.931299209595 9000 reviews parsed in chunk 77 time: 16923.489053964615 10000 reviews parsed in chunk 77 time: 16944.650870084763 1000 reviews parsed in chunk 78 time: 16969.010617256165 2000 reviews parsed in chunk 78 time: 16986.10706925392 3000 reviews parsed in chunk 78 time: 17005.11490726471 4000 reviews parsed in chunk 78 time: 17022.166852235794 5000 reviews parsed in chunk 78 time: 17039.544050216675 6000 reviews parsed in chunk 78 time: 17056.159992218018 7000 reviews parsed in chunk 78 time: 17076.774474143982 8000 reviews parsed in chunk 78 time: 17095.342581033707 9000 reviews parsed in chunk 78 time: 17116.1998462677 10000 reviews parsed in chunk 78 time: 17137.1433532238 1000 reviews parsed in chunk 79 time: 17163.2991271019 2000 reviews parsed in chunk 79 time: 17181.426252126694 3000 reviews parsed in chunk 79 time: 17202.197701215744 4000 reviews parsed in chunk 79 time: 17221.721781015396 5000 reviews parsed in chunk 79 time: 17242.17502117157 6000 reviews parsed in chunk 79 time: 17261.549044132233 7000 reviews parsed in chunk 79 time: 17282.5494222641 8000 reviews parsed in chunk 79 time: 17300.33540201187 9000 reviews parsed in chunk 79 time: 17326.88761496544 10000 reviews parsed in chunk 79 time: 17347.64376807213 1000 reviews parsed in chunk 80 time: 17366.84272003174 2000 reviews parsed in chunk 80 time: 17385.516919136047 3000 reviews parsed in chunk 80 time: 17402.95755505562 4000 reviews parsed in chunk 80 time: 17419.97353219986 5000 reviews parsed in chunk 80 time: 17439.903881072998 6000 reviews parsed in chunk 80 time: 17458.385172128677 7000 reviews parsed in chunk 80 time: 17475.424906015396 8000 reviews parsed in chunk 80 time: 17497.13298726082 9000 reviews parsed in chunk 80 time: 17514.692219257355 10000 reviews parsed in chunk 80 time: 17537.051753997803 1000 reviews parsed in chunk 81 time: 17559.490803956985 2000 reviews parsed in chunk 81 time: 17577.935737133026 3000 reviews parsed in chunk 81 time: 17597.397134065628 4000 reviews parsed in chunk 81 time: 17616.810395240784 5000 reviews parsed in chunk 81 time: 17642.001306295395 6000 reviews parsed in chunk 81 time: 17661.810717105865 7000 reviews parsed in chunk 81 time: 17681.089443206787 8000 reviews parsed in chunk 81 time: 17699.24983406067 9000 reviews parsed in chunk 81 time: 17714.215284109116 10000 reviews parsed in chunk 81 time: 17736.4173913002 1000 reviews parsed in chunk 82 time: 17758.79567217827 2000 reviews parsed in chunk 82 time: 17776.006927251816 3000 reviews parsed in chunk 82 time: 17793.765010356903 4000 reviews parsed in chunk 82 time: 17809.06805920601 5000 reviews parsed in chunk 82 time: 17832.044807195663 6000 reviews parsed in chunk 82 time: 17852.904201984406 7000 reviews parsed in chunk 82 time: 17872.048018217087 8000 reviews parsed in chunk 82 time: 17901.374695301056 9000 reviews parsed in chunk 82 time: 17921.841286182404 10000 reviews parsed in chunk 82 time: 17942.023577213287 1000 reviews parsed in chunk 83 time: 17966.01003909111 2000 reviews parsed in chunk 83 time: 17984.44114422798 3000 reviews parsed in chunk 83 time: 18007.69004011154 4000 reviews parsed in chunk 83 time: 18023.119982242584 5000 reviews parsed in chunk 83 time: 18040.682878017426 6000 reviews parsed in chunk 83 time: 18059.077553272247 7000 reviews parsed in chunk 83 time: 18079.431401014328 8000 reviews parsed in chunk 83 time: 18096.38137602806 9000 reviews parsed in chunk 83 time: 18113.454760074615 10000 reviews parsed in chunk 83 time: 18130.999127149582 1000 reviews parsed in chunk 84 time: 18148.73316001892 2000 reviews parsed in chunk 84 time: 18169.598650217056 3000 reviews parsed in chunk 84 time: 18191.424730300903 4000 reviews parsed in chunk 84 time: 18210.05843925476 5000 reviews parsed in chunk 84 time: 18226.8892390728 6000 reviews parsed in chunk 84 time: 18244.28283214569 7000 reviews parsed in chunk 84 time: 18264.21383213997 8000 reviews parsed in chunk 84 time: 18285.169545173645 9000 reviews parsed in chunk 84 time: 18306.816059350967 10000 reviews parsed in chunk 84 time: 18324.91323709488 1000 reviews parsed in chunk 85 time: 18350.806407928467 2000 reviews parsed in chunk 85 time: 18371.11451435089 3000 reviews parsed in chunk 85 time: 18389.530690193176 4000 reviews parsed in chunk 85 time: 18408.99975824356 5000 reviews parsed in chunk 85 time: 18426.557351350784 6000 reviews parsed in chunk 85 time: 18449.1361951828 7000 reviews parsed in chunk 85 time: 18468.348721265793 8000 reviews parsed in chunk 85 time: 18487.428405046463 9000 reviews parsed in chunk 85 time: 18504.853419065475 10000 reviews parsed in chunk 85 time: 18520.398055315018 1000 reviews parsed in chunk 86 time: 18541.338531255722 2000 reviews parsed in chunk 86 time: 18560.9890832901 3000 reviews parsed in chunk 86 time: 18579.416901111603 4000 reviews parsed in chunk 86 time: 18597.619375228882 5000 reviews parsed in chunk 86 time: 18618.245800971985 6000 reviews parsed in chunk 86 time: 18639.430909395218 7000 reviews parsed in chunk 86 time: 18661.298668146133 8000 reviews parsed in chunk 86 time: 18679.127370119095 9000 reviews parsed in chunk 86 time: 18702.991400003433 10000 reviews parsed in chunk 86 time: 18722.764097213745 1000 reviews parsed in chunk 87 time: 18749.768321037292 2000 reviews parsed in chunk 87 time: 18767.837739229202 3000 reviews parsed in chunk 87 time: 18788.960249185562 4000 reviews parsed in chunk 87 time: 18806.81977915764 5000 reviews parsed in chunk 87 time: 18827.675638198853 6000 reviews parsed in chunk 87 time: 18849.778457164764 7000 reviews parsed in chunk 87 time: 18868.3201379776 8000 reviews parsed in chunk 87 time: 18886.729813098907 9000 reviews parsed in chunk 87 time: 18906.14505815506 10000 reviews parsed in chunk 87 time: 18927.9814453125 1000 reviews parsed in chunk 88 time: 18951.229293107986 2000 reviews parsed in chunk 88 time: 18972.812814235687 3000 reviews parsed in chunk 88 time: 18988.12446331978 4000 reviews parsed in chunk 88 time: 19007.26150202751 5000 reviews parsed in chunk 88 time: 19029.90865111351 6000 reviews parsed in chunk 88 time: 19045.976892232895 7000 reviews parsed in chunk 88 time: 19062.12012910843 8000 reviews parsed in chunk 88 time: 19081.283268213272 9000 reviews parsed in chunk 88 time: 19099.124142169952 10000 reviews parsed in chunk 88 time: 19118.656689167023 1000 reviews parsed in chunk 89 time: 19140.345984220505 2000 reviews parsed in chunk 89 time: 19158.211527347565 3000 reviews parsed in chunk 89 time: 19175.687376260757 4000 reviews parsed in chunk 89 time: 19190.087254047394 5000 reviews parsed in chunk 89 time: 19207.75583910942 6000 reviews parsed in chunk 89 time: 19226.32416319847 7000 reviews parsed in chunk 89 time: 19243.432547092438 8000 reviews parsed in chunk 89 time: 19263.637775182724 9000 reviews parsed in chunk 89 time: 19285.61682200432 10000 reviews parsed in chunk 89 time: 19304.67928123474 1000 reviews parsed in chunk 90 time: 19327.2745821476 2000 reviews parsed in chunk 90 time: 19346.119744062424 3000 reviews parsed in chunk 90 time: 19362.327642202377 4000 reviews parsed in chunk 90 time: 19389.16126704216 5000 reviews parsed in chunk 90 time: 19407.579765081406 6000 reviews parsed in chunk 90 time: 19428.308703184128 7000 reviews parsed in chunk 90 time: 19455.39527821541 8000 reviews parsed in chunk 90 time: 19472.376559257507 9000 reviews parsed in chunk 90 time: 19489.606777191162 10000 reviews parsed in chunk 90 time: 19509.200946331024 1000 reviews parsed in chunk 91 time: 19531.68468117714 2000 reviews parsed in chunk 91 time: 19552.822721242905 3000 reviews parsed in chunk 91 time: 19572.374068021774 4000 reviews parsed in chunk 91 time: 19589.68765425682 5000 reviews parsed in chunk 91 time: 19607.982224225998 6000 reviews parsed in chunk 91 time: 19624.9943652153 7000 reviews parsed in chunk 91 time: 19650.789055109024 8000 reviews parsed in chunk 91 time: 19668.861831188202 9000 reviews parsed in chunk 91 time: 19686.639077186584 10000 reviews parsed in chunk 91 time: 19701.05848622322 1000 reviews parsed in chunk 92 time: 19722.669612169266 2000 reviews parsed in chunk 92 time: 19745.42847108841 3000 reviews parsed in chunk 92 time: 19765.446112155914 4000 reviews parsed in chunk 92 time: 19783.64008498192 5000 reviews parsed in chunk 92 time: 19802.095296144485 6000 reviews parsed in chunk 92 time: 19824.68865418434 7000 reviews parsed in chunk 92 time: 19843.017549276352 8000 reviews parsed in chunk 92 time: 19866.77135014534 9000 reviews parsed in chunk 92 time: 19883.086849212646 10000 reviews parsed in chunk 92 time: 19902.625185012817 1000 reviews parsed in chunk 93 time: 19926.380561351776 2000 reviews parsed in chunk 93 time: 19942.433094263077 3000 reviews parsed in chunk 93 time: 19958.035920143127 4000 reviews parsed in chunk 93 time: 19973.148626327515 5000 reviews parsed in chunk 93 time: 19991.73183321953 6000 reviews parsed in chunk 93 time: 20011.46251821518 7000 reviews parsed in chunk 93 time: 20036.955394268036 8000 reviews parsed in chunk 93 time: 20054.35497403145 9000 reviews parsed in chunk 93 time: 20072.511075258255 10000 reviews parsed in chunk 93 time: 20092.268032073975 1000 reviews parsed in chunk 94 time: 20110.45926308632 2000 reviews parsed in chunk 94 time: 20128.27276802063 3000 reviews parsed in chunk 94 time: 20151.344882249832 4000 reviews parsed in chunk 94 time: 20167.755988121033 5000 reviews parsed in chunk 94 time: 20187.044819116592 6000 reviews parsed in chunk 94 time: 20205.05735397339 7000 reviews parsed in chunk 94 time: 20220.919328212738 8000 reviews parsed in chunk 94 time: 20238.37049818039 9000 reviews parsed in chunk 94 time: 20254.570853233337 10000 reviews parsed in chunk 94 time: 20271.332415103912 1000 reviews parsed in chunk 95 time: 20293.787124156952 2000 reviews parsed in chunk 95 time: 20311.257691144943 3000 reviews parsed in chunk 95 time: 20328.244683027267 4000 reviews parsed in chunk 95 time: 20347.551254034042 5000 reviews parsed in chunk 95 time: 20362.667411088943 6000 reviews parsed in chunk 95 time: 20377.207592010498 7000 reviews parsed in chunk 95 time: 20392.060115098953 8000 reviews parsed in chunk 95 time: 20407.528887033463 9000 reviews parsed in chunk 95 time: 20423.056263923645 10000 reviews parsed in chunk 95 time: 20436.85669517517 1000 reviews parsed in chunk 96 time: 20452.660610198975 2000 reviews parsed in chunk 96 time: 20467.74929833412 3000 reviews parsed in chunk 96 time: 20481.84559226036 4000 reviews parsed in chunk 96 time: 20496.1255402565 5000 reviews parsed in chunk 96 time: 20512.596663236618 6000 reviews parsed in chunk 96 time: 20528.152697086334 7000 reviews parsed in chunk 96 time: 20542.90894627571 8000 reviews parsed in chunk 96 time: 20556.57521224022 9000 reviews parsed in chunk 96 time: 20571.285118103027 10000 reviews parsed in chunk 96 time: 20585.57377409935 1000 reviews parsed in chunk 97 time: 20604.191308021545 2000 reviews parsed in chunk 97 time: 20620.466699123383 3000 reviews parsed in chunk 97 time: 20636.55346107483 4000 reviews parsed in chunk 97 time: 20653.653450012207 5000 reviews parsed in chunk 97 time: 20670.277106285095 6000 reviews parsed in chunk 97 time: 20686.614187002182 7000 reviews parsed in chunk 97 time: 20703.052452087402 8000 reviews parsed in chunk 97 time: 20719.55308699608 9000 reviews parsed in chunk 97 time: 20735.859856128693 10000 reviews parsed in chunk 97 time: 20752.293530225754 1000 reviews parsed in chunk 98 time: 20770.71339917183 2000 reviews parsed in chunk 98 time: 20787.569498300552 3000 reviews parsed in chunk 98 time: 20803.762969255447 4000 reviews parsed in chunk 98 time: 20819.905877113342 5000 reviews parsed in chunk 98 time: 20836.531204223633 6000 reviews parsed in chunk 98 time: 20855.983055114746 7000 reviews parsed in chunk 98 time: 20871.95089817047 8000 reviews parsed in chunk 98 time: 20887.743999004364 9000 reviews parsed in chunk 98 time: 20904.02795124054 10000 reviews parsed in chunk 98 time: 20921.364195108414 1000 reviews parsed in chunk 99 time: 20939.957690238953 2000 reviews parsed in chunk 99 time: 20955.186669111252 3000 reviews parsed in chunk 99 time: 20969.236399173737 4000 reviews parsed in chunk 99 time: 20985.019869089127 5000 reviews parsed in chunk 99 time: 21000.3118891716 6000 reviews parsed in chunk 99 time: 21015.87318110466 7000 reviews parsed in chunk 99 time: 21032.089995145798 8000 reviews parsed in chunk 99 time: 21047.7855591774 9000 reviews parsed in chunk 99 time: 21062.592415094376 10000 reviews parsed in chunk 99 time: 21078.724135160446 1000 reviews parsed in chunk 100 time: 21097.96704530716 2000 reviews parsed in chunk 100 time: 21115.0389251709 3000 reviews parsed in chunk 100 time: 21131.45344734192 4000 reviews parsed in chunk 100 time: 21147.15246796608 5000 reviews parsed in chunk 100 time: 21161.121145248413 6000 reviews parsed in chunk 100 time: 21175.943430185318 7000 reviews parsed in chunk 100 time: 21188.98072910309
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) <ipython-input-7-8fbf603f8933> in <module>() 7 8 start = time.time() ----> 9 spacy_parse_store_from_dataframe(fname, review_df, nlp) 10 print('took:', time.time() - start, 'seconds') 11 ~/Code/Huygens/scale/scripts/text_tail_analysis.py in spacy_parse_store_from_dataframe(fname, df, nlp) 170 print(ti+1, 'reviews parsed in chunk', chunk, '\ttime:', time.time() - start_time) 171 with open(chunk_fname, 'wb') as fh: --> 172 fh.write(doc_bin.to_bytes()) 173 174 ~/Library/Python/3.6/lib/python/site-packages/spacy/tokens/_serialize.py in to_bytes(self) 141 msg = { 142 "attrs": self.attrs, --> 143 "tokens": numpy.vstack(self.tokens).tobytes("C"), 144 "spaces": numpy.vstack(self.spaces).tobytes("C"), 145 "lengths": numpy.asarray(lengths, dtype="int32").tobytes("C"), ~/Library/Python/3.6/lib/python/site-packages/numpy/core/shape_base.py in vstack(tup) 281 """ 282 _warn_for_nonsequence(tup) --> 283 return _nx.concatenate([atleast_2d(_m) for _m in tup], 0) 284 285 ValueError: need at least one array to concatenate