Merge pull request #1381 from ebocher/some_fixes

Read the cpg file attached to the shapefile

docs/CHANGELOG.md

@@ -1 +1,3 @@
 ## Changelog for v2.2.2
+
+- Read CPG file attached to the shapefile

h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java

@@ -229,6 +229,8 @@ public  String[]  importFile(Connection connection, String tableReference, File
                 stmt.execute("DROP TABLE IF EXISTS " + outputTable);
                 stmt.close();
             }
+            //Check if a cpg file exists
+            //TODO:
             DBFDriver dbfDriver = new DBFDriver();
             dbfDriver.initDriverFromFile(fileName, options);
             DbaseFileHeader dbfHeader = dbfDriver.getDbaseFileHeader();

h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileHeader.java

@@ -503,15 +503,49 @@ public void readHeader(FileChannel channel,String forceEncoding) throws IOExcept
 
 		// skip / skip thesreserved bytes in the header.
 		in.skip(17);
-        // read Language driver
-        byte lngDriver = in.get();
-        String encoding = CODE_PAGE_ENCODING.get(lngDriver);
-        if(encoding!=null && forceEncoding == null) {
-            this.fileEncoding = encoding;
-        }
-        // skip reserved
-        in.skip(2);
+		// read Language driver
+		byte lngDriver = in.get();
+		String encoding = CODE_PAGE_ENCODING.get(lngDriver);
+		// skip reserved
+		in.skip(2);
+		//FROM : https://github.com/OSGeo/gdal/blob/master/ogr/ogrsf_frmts/shape/ogrshapelayer.cpp#L526
+		if(forceEncoding!=null){
+			forceEncoding = forceEncoding.toUpperCase();
+			try {
+				//Only code page
+				int codepage = Integer.valueOf(forceEncoding);
+				if ((codepage >= 437 && codepage <= 950) || (codepage >= 1250 && codepage <= 1258)) {
+					this.fileEncoding = new StringBuffer("CP").append(forceEncoding).toString();
+				}
+				else if (forceEncoding.startsWith("8859"))
+				{
+					if(forceEncoding.startsWith("-", 4)){
+						this.fileEncoding ="ISO-8859-5";
+					}else{
+						this.fileEncoding ="ISO-8859-4";
+					}
+				}
+				else if (forceEncoding.startsWith("UTF-8")||forceEncoding.startsWith("UTF8"))
+				{
+					this.fileEncoding ="UTF-8";
+				}
+				else if (forceEncoding.startsWith( "ANSI 1251")) {
+					this.fileEncoding = "CP1251";
+				}
+				else
+				{
+					// Try just using the CPG value directly.  Works for stuff like Big5.
+					fileEncoding = forceEncoding;
+				}
+			}catch (NumberFormatException ex){
+				//Nothing to do
+			}
 
+		}else {
+			if(encoding!=null){
+				this.fileEncoding = encoding;
+			}
+		}
 		// calculate the number of Fields in the header
 		fieldCnt = (headerLength - FILE_DESCRIPTOR_SIZE - 1)
 				/ FILE_DESCRIPTOR_SIZE;

h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java

@@ -32,6 +32,7 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * Merge ShapeFileReader and DBFReader.
@@ -57,6 +58,7 @@ public class SHPDriver implements FileDriver {
     private int geometryFieldIndex = 0;
     private ShapeType shapeType;
     public File prjFile;
+    public File cpgFile;
     private int srid =0;
 
 
@@ -168,6 +170,9 @@ else if(path.equals(fileNamePrefix+".dbf")){
                 else if(path.equals(fileNamePrefix+".prj")){
                     prjFile = entry.toFile();
                     return true;
+                } else if(path.equals(fileNamePrefix+".cpg")){
+                    cpgFile = entry.toFile();
+                    return true;
                 }
                 return false;
             }
@@ -180,6 +185,14 @@ else if(path.equals(fileNamePrefix+".prj")){
             } 
         }
         if(dbfFile != null) {
+            //Read the CPG file if exists
+            if(cpgFile!=null){
+                BufferedReader br = Files.newBufferedReader(cpgFile.toPath());
+                String codePage;
+                if ((codePage = br.readLine()) != null && forceEncoding==null) {
+                    forceEncoding = codePage.trim();
+                }
+            }
             dbfDriver.initDriverFromFile(dbfFile, forceEncoding);
         } else {
             throw new IllegalArgumentException("DBF File not found");

h2gis-functions/src/test/java/org/h2gis/functions/io/shp/SHPEngineTest.java

@@ -449,4 +449,32 @@ public void linkedShpSpatialIndexFlatQueryTest() throws SQLException {
             assertTrue(rs.getString(1).contains("PK_INDEX"), "Expected contains PK_INDEX but result is " + rs.getString(1));
         }
     }
+
+    @Test
+    public void readSHPWithCPGFileTest() throws SQLException {
+        Statement st = connection.createStatement();
+        st.execute("drop table if exists shptable");
+        st.execute("CALL FILE_TABLE('"+SHPEngineTest.class.getResource("urock_buildings.shp").getPath()+"', 'SHPtable');");
+        try ( // Query declared Table columns
+              ResultSet rs = st.executeQuery("SELECT * FROM SHPTABLE where objektiden= '71193131-5b61-4d65-a661-fefb173bfd86';")) {
+            assertTrue(rs.next());
+            assertEquals("71193131-5b61-4d65-a661-fefb173bfd86", rs.getString("objektiden"));
+            assertEquals("Bostad;Småhus friliggande",rs.getString("andamal1"));
+        }
+        st.execute("drop table shptable");
+    }
+
+    @Test
+    public void readSHPWithCPGFordeTest() throws SQLException {
+        Statement st = connection.createStatement();
+        st.execute("drop table if exists shptable");
+        st.execute("CALL SHPREAD( '"+SHPEngineTest.class.getResource("urock_buildings.shp").getPath()+"', 'SHPtable','windows-1252');");
+        try ( // Query declared Table columns
+              ResultSet rs = st.executeQuery("SELECT * FROM SHPTABLE where objektiden= '3110e3c4-638c-485e-b5fe-a827ebafd071';")) {
+            assertTrue(rs.next());
+            assertEquals("3110e3c4-638c-485e-b5fe-a827ebafd071", rs.getString("objektiden"));
+            assertEquals("Industri;Tillverkning",rs.getString("andamal1"));
+        }
+        st.execute("drop table shptable");
+    }
 }

h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.cpg

@@ -0,0 +1 @@
+1252

h2gis-functions/src/test/resources/org/h2gis/functions/io/shp/urock_buildings.prj

@@ -0,0 +1 @@
+PROJCS["SWEREF99_12_00",GEOGCS["GCS_SWEREF99",DATUM["D_SWEREF99",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",150000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",12.0],PARAMETER["Scale_Factor",1.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]