diff --git a/app/uploaders/zizia/csv_manifest_validator.rb b/app/uploaders/zizia/csv_manifest_validator.rb index 6b4732e..79c198b 100644 --- a/app/uploaders/zizia/csv_manifest_validator.rb +++ b/app/uploaders/zizia/csv_manifest_validator.rb @@ -57,7 +57,7 @@ def valid_headers end def parse_csv - @rows = CSV.read(csv_file.path) + @rows = CSV.read(csv_file.path).reject { |x| x.empty? || x.all?(nil) } @headers = @rows.first || [] @transformed_headers = @headers.map { |header| header.downcase.strip } rescue diff --git a/spec/dummy/spec/fixtures/csv_import/good/Postcards_Minneapolis_w_collection.csv b/spec/dummy/spec/fixtures/csv_import/good/Postcards_Minneapolis_w_collection.csv index 708e48f..d8023d5 100644 --- a/spec/dummy/spec/fixtures/csv_import/good/Postcards_Minneapolis_w_collection.csv +++ b/spec/dummy/spec/fixtures/csv_import/good/Postcards_Minneapolis_w_collection.csv @@ -1,8 +1,11 @@ object type,Title,Resource Type,Creator,Contributor,Description,Keyword,License,Rights Statement,Publisher,Date Created,Subject,Language,Identifier,Based Near,Related Url,Bibliographic Citation,Source,Visibility,files,deduplication_key w,"Canoeing on Lake Harriet, Minneapolis with color bars",Image,Minneapolis Selling Company,"Bussey, Mark H.","Postcard depicting Lake Harriet in Minneapolis, MN with color bars",postcard|~|Minneapolis|~|Lake Harriet,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,Minneapolis Selling Company,1900-1945,Postcards,English,PC-001,,,,,open,MN-02.png,PC-001 + +,,,,,,,,,,,,,,,,,,,, w,"Canoeing on Lake Harriet, Minneapolis",Image,Minneapolis Selling Company,"Bussey, Mark H.","Postcard depicting Lake Harriet in Minnneapolis, MN",postcard|~|Minneapolis|~|Lake Harriet,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,Minneapolis Selling Company,1900-1945,Postcards,English,PC-002,,,,,open,MN-02 1.png|~|MN-02 2.png,PC-002 w,"Lake Harriet, Minneapolis, Minn",Image,Bloom Bros Co.,"Bussey, Mark H.",Postcard depicting Lake Harriet,postcard|~|Minneapolis|~|Lake Harriet,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,Bloom Bros Co.,1900-1945,Postcards,English,PC-003,,,,,open,MN-02 3.png|~|MN-02 4.png,PC-003 w,"Art Museum, Minneapolis, Minn",Image,Bloom Bros Co.,"Bussey, Mark H.",Postcard depicting The Minneapolis Institute of Arts,postcard|~|Minneapolis|~|Art Museum,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,Bloom Bros Co.,1900-1945,Postcards,English,PC-004,,,,,open,MN-02 5.png|~|MN-02 6.png,PC-004 + w,"The Milling District by Moonlight over the Ice-bound Mississippi, Minneapolis, Minn",Image,VO Mammon Publishing,"Bussey, Mark H.",Postcard depicting the Milling District of Minneapolis at night,postcard|~|Minneapolis|~|Milling District,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,VO Mamon Publishing,1900-1945,Postcards,English,PC-005,,,,,open,MN-02 7.png|~|MN-02 8.png,PC-005 w,"Business District, Minneapolis, MN",Image,The Rotograph Co. N.Y City,"Bussey, Mark H.",Postcard depicting the Business District of Minneapolis,postcard|~|Minneapolis|~|Business District,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,The Rotograph Co. N.Y City,1900-1945,Postcards,English,PC-006,,,,,open,MN-02 9.png|~|MN-02 10.png,PC-006 w,Minnehaha Falls,Image,Unknown,"Bussey, Mark H.",Postcard depicting Minnehaha Falls in Minneapolis,postcard|~|Minneapolis|~|Minnehaha Falls,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,,1900-1945,Postcards,English,PC-007,,,,,open,MN-02 11.png|~|MN-02 12.png,PC-007 @@ -11,4 +14,4 @@ w,"Twin City R. T Co. Power House and Dam, Mississippi River",Image,Curt Teich & w,"Hennepin Ave from 8th St., Minneapolis",Image,"Grogan Photo System, Inc","Bussey, Mark H.",Postcard depicting Hennepin Ave. in Minneapolis,postcard|~|Minneapolis|~|Hennepin Ave,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,"Grogan Photo System, Inc",1900-1945,Postcards,English,PC-010,,,,,open,MN-02 17.png|~|MN-02 18.png,PC-010 w,"Bird's Eye View of Minneapolis, Minn., By Night",Image,Bloom Bros Co.,"Bussey, Mark H.",Postcard depicting Minneapolis from a bird's eye view,postcard|~|Minneapolis|~|downtown,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,Bloom Bros Co.,1900-1945,Postcards,English,PC-011,,,,,open,MN-02 19.png|~|MN-02 20.png,PC-011 w,"Third Ave. Bridge, Minneapolis",Image,"Grogan Photo System, Inc","Bussey, Mark H.",Postcard depicting Third Ave. Bridge in Minneapolis,postcard|~|Minneapolis|~|Third Avenue|~|bridge,http://creativecommons.org/publicdomain/mark/1.0/,http://rightsstatements.org/vocab/NoC-OKLR/1.0/,"Grogan Photo System, Inc",1900-1945,Postcards,English,PC-012,,,,,open,MN-02 21.png|~|MN-02 22.png,PC-012 -c,Minneapolis postcards,,,,,,,,,,,,,,,,,,, \ No newline at end of file +c,Minneapolis postcards,,,,,,,,,,,,,,,,,,, diff --git a/spec/dummy/spec/fixtures/csv_import/good/all_fields_only_new.csv b/spec/dummy/spec/fixtures/csv_import/good/all_fields_only_new.csv index 6aaf299..3c3b252 100644 --- a/spec/dummy/spec/fixtures/csv_import/good/all_fields_only_new.csv +++ b/spec/dummy/spec/fixtures/csv_import/good/all_fields_only_new.csv @@ -1,3 +1,6 @@ identifier,license,deduplication_key,visibility,location,keyword,rights statement,creator,title,files + abc/123,https://creativecommons.org/licenses/by/4.0/,abc/123,PUBlic,http://www.geonames.org/5667009/montana.html|~|http://www.geonames.org/6252001/united-states.html,Clothing stores $z California $z Los Angeles|~|Interior design $z California $z Los Angeles,http://rightsstatements.org/vocab/InC/1.0/,"Connell, Will, $d 1898-1961","Interior view of The Bachelors haberdashery designed by Julius Ralph Davidson, Los Angeles, circa 1929",dog.jpg + +,,,,,,,,, abc/456,https://creativecommons.org/licenses/by/4.0/,abc/456,PUBlic,http://www.geonames.org/5667009/montana.html|~|http://www.geonames.org/6252001/united-states.html,Clothing stores $z California $z Los Angeles|~|Interior design $z California $z Los Angeles,http://rightsstatements.org/vocab/InC/1.0/,"Connell, Will, $d 1898-1961","Interior view of The Bachelors patisserie designed by Julius Ralph Davidson, Los Angeles, circa 1929",cat.jpg diff --git a/spec/zizia/csv_parser_spec.rb b/spec/zizia/csv_parser_spec.rb index 9820ef4..4e5b3c3 100644 --- a/spec/zizia/csv_parser_spec.rb +++ b/spec/zizia/csv_parser_spec.rb @@ -14,6 +14,7 @@ title,description,date created The Moomins and the Great Flood,"The Moomins and the Great Flood (Swedish: Småtrollen och den stora översvämningen, literally The Little Trolls and the Great Flood) is a book written by Finnish author Tove Jansson in 1945, during the end of World War II. It was the first book to star the Moomins, but is often seen as a prelude to the main Moomin books, as most of the main characters are introduced in the next book.",1945 +,, Comet in Moominland,"Comet in Moominland is the second in Tove Jansson's series of Moomin books. Published in 1946, it marks the first appearance of several main characters, like Snufkin and the Snork Maiden.",1946 EOS end