[{"data":1,"prerenderedAt":731},["ShallowReactive",2],{"content-\u002Fuse-cases\u002Fcleaning-csv-files":3,"use-cases-nav":726},{"id":4,"title":5,"author":6,"body":7,"date":6,"description":719,"extension":720,"meta":721,"navigation":207,"order":36,"path":722,"seo":723,"stem":724,"version":6,"__hash__":725},"content\u002Fuse-cases\u002Fcleaning-csv-files.md","Cleaning .csv files",null,{"type":8,"value":9,"toc":714},"minimark",[10,14,17,22,144,147,169,173,176,427,431,434,699,702,710],[11,12,13],"p",{},"As a database and software consultant I frequently face the requirement of\ncleaning and migrating legacy data to a new database structure. Such data\noften comes from excel or as CSV files extracted from a previous database.\nAn invariant is that legacy data is noisy and must alway be cleaned first.",[11,15,16],{},"So suppose a CSV file, with noisy data (missing required fields, wrong dates,\netc.). Your job is to split that CSV file: valid records on STDOUT, invalid\nones on STDERR with an explanation.",[18,19,21],"h2",{"id":20},"the-data","The data",[23,24,29],"pre",{"className":25,"code":26,"language":27,"meta":28,"style":28},"language-csv shiki shiki-themes github-light github-dark","id,name,birthdate,score\n1,Aaron,2013-10-28,5\n2,Courtney,2013-26-08,5\n3,Connor,2014-10-11,8\n4,Meredith,2014-10-15,22\n5,Kristen,2014-03-28,8\n6,Nissim,2014-06-05,\n...\n","csv","",[30,31,32,53,68,82,97,112,126,138],"code",{"__ignoreMap":28},[33,34,37,41,45,49],"span",{"class":35,"line":36},"line",1,[33,38,40],{"class":39},"sVt8B","id,",[33,42,44],{"class":43},"szBVR","name,",[33,46,48],{"class":47},"sScJk","birthdate,",[33,50,52],{"class":51},"sJ8bj","score\n",[33,54,56,59,62,65],{"class":35,"line":55},2,[33,57,58],{"class":39},"1,",[33,60,61],{"class":43},"Aaron,",[33,63,64],{"class":47},"2013-10-28,",[33,66,67],{"class":51},"5\n",[33,69,71,74,77,80],{"class":35,"line":70},3,[33,72,73],{"class":39},"2,",[33,75,76],{"class":43},"Courtney,",[33,78,79],{"class":47},"2013-26-08,",[33,81,67],{"class":51},[33,83,85,88,91,94],{"class":35,"line":84},4,[33,86,87],{"class":39},"3,",[33,89,90],{"class":43},"Connor,",[33,92,93],{"class":47},"2014-10-11,",[33,95,96],{"class":51},"8\n",[33,98,100,103,106,109],{"class":35,"line":99},5,[33,101,102],{"class":39},"4,",[33,104,105],{"class":43},"Meredith,",[33,107,108],{"class":47},"2014-10-15,",[33,110,111],{"class":51},"22\n",[33,113,115,118,121,124],{"class":35,"line":114},6,[33,116,117],{"class":39},"5,",[33,119,120],{"class":43},"Kristen,",[33,122,123],{"class":47},"2014-03-28,",[33,125,96],{"class":51},[33,127,129,132,135],{"class":35,"line":128},7,[33,130,131],{"class":39},"6,",[33,133,134],{"class":43},"Nissim,",[33,136,137],{"class":47},"2014-06-05,\n",[33,139,141],{"class":35,"line":140},8,[33,142,143],{"class":39},"...\n",[11,145,146],{},"Given that,",[148,149,150,162],"ul",{},[151,152,153,154,157,158,161],"li",{},"Some dates are known to be wrong. It was a text field and some people\ninverted the month and the day, leading to dates such as ",[30,155,156],{},"2013-26-08","\ninstead of ",[30,159,160],{},"2013-08-26"," (see row 2).",[151,163,164,165,168],{},"The ",[30,166,167],{},"score"," field is required and supposed to be between 0 and 10. In\npractice it is sometimes missing or bigger than 10 (see rows 4 and 6).",[18,170,172],{"id":171},"finition-schema-at-the-rescue","Finition schema at the rescue",[11,174,175],{},"Let write a schema that will coerce the data and detect invalid values.",[23,177,181],{"className":178,"code":179,"language":180,"meta":28,"style":28},"language-ruby shiki shiki-themes github-light github-dark","# String is an alias for Ruby's String\nString  = .String\n\n# We'll have to coerce Integer literals from ruby String\nInteger = \u003Cliteral> .String \\( s | Integer(s) )       \\( i | i.to_s    )\n\n# We'll have to coerce Date literals from iso8601 String \nDate    = \u003Cliteral> .String \\( s | Date.iso8601(s) )  \\( d | d.iso8601 )\n\n# Valid scores are integers between 0 and 10 inclusive\nScore = Integer( i | valid: i >= 0 and i \u003C= 10 )\n\n# One CSV row should meet the following tuple type\n{\n  id:   Integer,\n  name: String,\n  birthdate: Date,\n  score: Score\n}\n","ruby",[30,182,183,188,203,209,214,258,262,267,311,316,322,362,367,373,379,391,402,412,421],{"__ignoreMap":28},[33,184,185],{"class":35,"line":36},[33,186,187],{"class":51},"# String is an alias for Ruby's String\n",[33,189,190,194,197,200],{"class":35,"line":55},[33,191,193],{"class":192},"sj4cs","String",[33,195,196],{"class":43},"  =",[33,198,199],{"class":39}," .",[33,201,202],{"class":192},"String\n",[33,204,205],{"class":35,"line":70},[33,206,208],{"emptyLinePlaceholder":207},true,"\n",[33,210,211],{"class":35,"line":84},[33,212,213],{"class":51},"# We'll have to coerce Integer literals from ruby String\n",[33,215,216,219,222,225,228,231,233,235,238,241,244,247,249,252,255],{"class":35,"line":99},[33,217,218],{"class":192},"Integer",[33,220,221],{"class":43}," =",[33,223,224],{"class":43}," \u003C",[33,226,227],{"class":39},"literal",[33,229,230],{"class":43},">",[33,232,199],{"class":39},[33,234,193],{"class":192},[33,236,237],{"class":39}," \\( s ",[33,239,240],{"class":43},"|",[33,242,243],{"class":192}," Integer",[33,245,246],{"class":39},"(s) )       \\( i ",[33,248,240],{"class":43},[33,250,251],{"class":39}," i.",[33,253,254],{"class":47},"to_s",[33,256,257],{"class":39},"    )\n",[33,259,260],{"class":35,"line":114},[33,261,208],{"emptyLinePlaceholder":207},[33,263,264],{"class":35,"line":128},[33,265,266],{"class":51},"# We'll have to coerce Date literals from iso8601 String \n",[33,268,269,272,275,277,279,281,283,285,287,289,292,295,298,301,303,306,308],{"class":35,"line":140},[33,270,271],{"class":192},"Date",[33,273,274],{"class":43},"    =",[33,276,224],{"class":43},[33,278,227],{"class":39},[33,280,230],{"class":43},[33,282,199],{"class":39},[33,284,193],{"class":192},[33,286,237],{"class":39},[33,288,240],{"class":43},[33,290,291],{"class":192}," Date",[33,293,294],{"class":39},".",[33,296,297],{"class":47},"iso8601",[33,299,300],{"class":39},"(s) )  \\( d ",[33,302,240],{"class":43},[33,304,305],{"class":39}," d.",[33,307,297],{"class":47},[33,309,310],{"class":39}," )\n",[33,312,314],{"class":35,"line":313},9,[33,315,208],{"emptyLinePlaceholder":207},[33,317,319],{"class":35,"line":318},10,[33,320,321],{"class":51},"# Valid scores are integers between 0 and 10 inclusive\n",[33,323,325,328,330,332,335,337,340,343,346,349,352,354,357,360],{"class":35,"line":324},11,[33,326,327],{"class":192},"Score",[33,329,221],{"class":43},[33,331,243],{"class":192},[33,333,334],{"class":39},"( i ",[33,336,240],{"class":43},[33,338,339],{"class":192}," valid:",[33,341,342],{"class":39}," i ",[33,344,345],{"class":43},">=",[33,347,348],{"class":192}," 0",[33,350,351],{"class":43}," and",[33,353,342],{"class":39},[33,355,356],{"class":43},"\u003C=",[33,358,359],{"class":192}," 10",[33,361,310],{"class":39},[33,363,365],{"class":35,"line":364},12,[33,366,208],{"emptyLinePlaceholder":207},[33,368,370],{"class":35,"line":369},13,[33,371,372],{"class":51},"# One CSV row should meet the following tuple type\n",[33,374,376],{"class":35,"line":375},14,[33,377,378],{"class":39},"{\n",[33,380,382,385,388],{"class":35,"line":381},15,[33,383,384],{"class":192},"  id:",[33,386,387],{"class":192},"   Integer",[33,389,390],{"class":39},",\n",[33,392,394,397,400],{"class":35,"line":393},16,[33,395,396],{"class":192},"  name:",[33,398,399],{"class":192}," String",[33,401,390],{"class":39},[33,403,405,408,410],{"class":35,"line":404},17,[33,406,407],{"class":192},"  birthdate:",[33,409,291],{"class":192},[33,411,390],{"class":39},[33,413,415,418],{"class":35,"line":414},18,[33,416,417],{"class":192},"  score:",[33,419,420],{"class":192}," Score\n",[33,422,424],{"class":35,"line":423},19,[33,425,426],{"class":39},"}\n",[18,428,430],{"id":429},"making-the-split-using-finitio-rb","Making the split using Finitio(-rb)",[11,432,433],{},"Here we go! Dress each CSV line using the schema, rescue on a type error:",[23,435,437],{"className":178,"code":436,"language":180,"meta":28,"style":28},"require 'finitio'\nrequire 'csv'\n\n# Let load the schema\nschema = Finitio.parse(File.read('schema.fio'))\n\n# For each CSV row\nFile.open('data.csv', 'r') do |io|\n  csv = ::CSV.new io, headers: true\n  csv.each do |row|\n    tuple = row.to_hash\n\n    # Dress the tuple, on success puts it on STDOUT with coerced\n    # values. On error display it with the error message on STDERR\n    begin\n      puts schema.dress(tuple).inspect\n    rescue Finitio::TypeError => ex\n      puts \"Skipping `#{row.to_s.strip}`: #{ex.message}\"\n    end\n\n  end\nend\n",[30,438,439,448,455,459,464,499,503,508,537,564,578,591,595,600,605,610,627,643,677,682,687,693],{"__ignoreMap":28},[33,440,441,444],{"class":35,"line":36},[33,442,443],{"class":43},"require",[33,445,447],{"class":446},"sZZnC"," 'finitio'\n",[33,449,450,452],{"class":35,"line":55},[33,451,443],{"class":43},[33,453,454],{"class":446}," 'csv'\n",[33,456,457],{"class":35,"line":70},[33,458,208],{"emptyLinePlaceholder":207},[33,460,461],{"class":35,"line":84},[33,462,463],{"class":51},"# Let load the schema\n",[33,465,466,470,472,475,477,480,483,486,488,491,493,496],{"class":35,"line":99},[33,467,469],{"class":468},"s4XuR","schema",[33,471,221],{"class":43},[33,473,474],{"class":192}," Finitio",[33,476,294],{"class":39},[33,478,479],{"class":47},"parse",[33,481,482],{"class":39},"(",[33,484,485],{"class":192},"File",[33,487,294],{"class":39},[33,489,490],{"class":47},"read",[33,492,482],{"class":39},[33,494,495],{"class":446},"'schema.fio'",[33,497,498],{"class":39},"))\n",[33,500,501],{"class":35,"line":114},[33,502,208],{"emptyLinePlaceholder":207},[33,504,505],{"class":35,"line":128},[33,506,507],{"class":51},"# For each CSV row\n",[33,509,510,512,514,517,519,522,525,528,531,534],{"class":35,"line":140},[33,511,485],{"class":192},[33,513,294],{"class":39},[33,515,516],{"class":192},"open",[33,518,482],{"class":39},[33,520,521],{"class":446},"'data.csv'",[33,523,524],{"class":39},", ",[33,526,527],{"class":446},"'r'",[33,529,530],{"class":39},") ",[33,532,533],{"class":43},"do",[33,535,536],{"class":39}," |io|\n",[33,538,539,542,544,547,550,552,555,558,561],{"class":35,"line":313},[33,540,541],{"class":468},"  csv",[33,543,221],{"class":43},[33,545,546],{"class":39}," ::",[33,548,549],{"class":192},"CSV",[33,551,294],{"class":39},[33,553,554],{"class":43},"new",[33,556,557],{"class":39}," io, ",[33,559,560],{"class":192},"headers:",[33,562,563],{"class":192}," true\n",[33,565,566,569,572,575],{"class":35,"line":318},[33,567,568],{"class":39},"  csv.",[33,570,571],{"class":47},"each",[33,573,574],{"class":43}," do",[33,576,577],{"class":39}," |row|\n",[33,579,580,583,585,588],{"class":35,"line":324},[33,581,582],{"class":468},"    tuple",[33,584,221],{"class":43},[33,586,587],{"class":39}," row.",[33,589,590],{"class":47},"to_hash\n",[33,592,593],{"class":35,"line":364},[33,594,208],{"emptyLinePlaceholder":207},[33,596,597],{"class":35,"line":369},[33,598,599],{"class":51},"    # Dress the tuple, on success puts it on STDOUT with coerced\n",[33,601,602],{"class":35,"line":375},[33,603,604],{"class":51},"    # values. On error display it with the error message on STDERR\n",[33,606,607],{"class":35,"line":381},[33,608,609],{"class":43},"    begin\n",[33,611,612,615,618,621,624],{"class":35,"line":393},[33,613,614],{"class":192},"      puts",[33,616,617],{"class":39}," schema.",[33,619,620],{"class":47},"dress",[33,622,623],{"class":39},"(tuple).",[33,625,626],{"class":47},"inspect\n",[33,628,629,632,634,637,640],{"class":35,"line":404},[33,630,631],{"class":43},"    rescue",[33,633,474],{"class":192},[33,635,636],{"class":39},"::",[33,638,639],{"class":192},"TypeError",[33,641,642],{"class":39}," => ex\n",[33,644,645,647,650,653,655,657,660,663,666,669,672,674],{"class":35,"line":414},[33,646,614],{"class":192},[33,648,649],{"class":446}," \"Skipping `",[33,651,652],{"class":446},"#{row.",[33,654,254],{"class":47},[33,656,294],{"class":446},[33,658,659],{"class":47},"strip",[33,661,662],{"class":446},"}",[33,664,665],{"class":446},"`: ",[33,667,668],{"class":446},"#{ex.",[33,670,671],{"class":47},"message",[33,673,662],{"class":446},[33,675,676],{"class":446},"\"\n",[33,678,679],{"class":35,"line":423},[33,680,681],{"class":43},"    end\n",[33,683,685],{"class":35,"line":684},20,[33,686,208],{"emptyLinePlaceholder":207},[33,688,690],{"class":35,"line":689},21,[33,691,692],{"class":43},"  end\n",[33,694,696],{"class":35,"line":695},22,[33,697,698],{"class":43},"end\n",[11,700,701],{},"Example output:",[23,703,708],{"className":704,"code":706,"language":707},[705],"language-text","{:id=>1, :name=>\"Aaron\", :birthdate=>#\u003CDate: 2013-10-28 ((2456594j,0s,0n),+0s,2299161j)>, :score=>5}\nSkipping `2,Courtney,2013-26-08,5`: Invalid value `2013-26-08` for Date\n{:id=>3, :name=>\"Connor\", :birthdate=>#\u003CDate: 2014-10-11 ((2456942j,0s,0n),+0s,2299161j)>, :score=>8}\nSkipping `4,Meredith,2014-10-15,22`: Invalid value `22` for Score (not valid)\n{:id=>5, :name=>\"Kristen\", :birthdate=>#\u003CDate: 2014-03-28 ((2456745j,0s,0n),+0s,2299161j)>, :score=>8}\nSkipping `6,Nissim,2014-06-05,`: Invalid value `nil` for Score\n{:id=>7, :name=>\"Donna\", :birthdate=>#\u003CDate: 2013-03-29 ((2456381j,0s,0n),+0s,2299161j)>, :score=>2}\n{:id=>8, :name=>\"Jenette\", :birthdate=>#\u003CDate: 2014-04-04 ((2456752j,0s,0n),+0s,2299161j)>, :score=>2}\n{:id=>9, :name=>\"Mason\", :birthdate=>#\u003CDate: 2014-08-05 ((2456875j,0s,0n),+0s,2299161j)>, :score=>8}\nSkipping `10,Yolanda,2014-03-25,77`: Invalid value `77` for Score (not valid)\n{:id=>11, :name=>\"Edan\", :birthdate=>#\u003CDate: 2015-02-18 ((2457072j,0s,0n),+0s,2299161j)>, :score=>8}\n","text",[30,709,706],{"__ignoreMap":28},[711,712,713],"style",{},"html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}",{"title":28,"searchDepth":70,"depth":70,"links":715},[716,717,718],{"id":20,"depth":55,"text":21},{"id":171,"depth":55,"text":172},{"id":429,"depth":55,"text":430},"Splitting a noisy CSV file into valid and invalid records with Finitio-rb.","md",{},"\u002Fuse-cases\u002Fcleaning-csv-files",{"title":5,"description":719},"use-cases\u002Fcleaning-csv-files","rSo0kv88jimTuM-9e7U2kUOU23iY5LqOnnYm4vBYMts",[727,730],{"path":728,"title":729},"\u002Fuse-cases","Use cases",{"path":722,"title":5},1786026251776]