jnr fuse support set max number of files open simultaneously (#36)

* jnr fuse support set max number of files open simultaneously * fix review issues
opendataio · Aug 18, 2021 · 2deb625 · 2deb625
1 parent 6804897
commit 2deb625
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 8 deletions.
diff --git a/...ain/java/hcfsfuse/fuse/AuthConstants.java → src/main/java/hcfsfuse/fuse/Constants.java b/...ain/java/hcfsfuse/fuse/AuthConstants.java → src/main/java/hcfsfuse/fuse/Constants.java
@@ -3,10 +3,12 @@
 /**
  * Convenience class of auth options.
  */
-public class AuthConstants {
+public class Constants {
   public static final String AUTH_POLICY = "hcfs.fuse.auth.policy";
   public static final String AUTH_POLICY_CUSTOM = "custom";
   public static final String AUTH_POLICY_CUSTOM_USER = "hcfs.fuse.auth.custom.user";
   public static final String AUTH_POLICY_CUSTOM_GROUP = "hcfs.fuse.auth.custom.group";
   public static final String AUTH_POLICY_IGNORE_MKDIR_GROUP = "hcfs.fuse.ignore.mkdir.group";
+  // jnr fuse max number of files that can be opened simultaneously
+  public static final String JNR_OPEN_FILE_CONCURRENT = "hcfs.fuse.jnr.open.file.concurrent";
 }
diff --git a/src/main/java/hcfsfuse/fuse/HCFSFuseFileSystem.java b/src/main/java/hcfsfuse/fuse/HCFSFuseFileSystem.java
@@ -1,5 +1,7 @@
 package hcfsfuse.fuse;
 
+import static hcfsfuse.fuse.Constants.JNR_OPEN_FILE_CONCURRENT;
+
 import alluxio.collections.IndexDefinition;
 import alluxio.collections.IndexedSet;
 import alluxio.fuse.AlluxioFuseUtils;
@@ -68,6 +70,7 @@ public class HCFSFuseFileSystem extends FuseStubFS {
   private AtomicLong mNextOpenFileId = new AtomicLong(0);
   private final LoadingCache<String, Path> mPathResolverCache;
   private final Configuration mConfiguration;
+  private final int openoncurrent;
 
   // Open file managements
   private static final IndexDefinition<OpenFileEntry<FSDataInputStream, FSDataOutputStream>, Long>
@@ -104,6 +107,7 @@ public HCFSFuseFileSystem(FileSystem fs, FuseOptions fuseOptions,
         .build(new PathCacheLoader());
     mOpenFiles = new IndexedSet<>(ID_INDEX, PATH_INDEX);
     mConfiguration = conf;
+    openoncurrent = mConfiguration.getInt(JNR_OPEN_FILE_CONCURRENT, MAX_OPEN_FILES);
   }
 
   @Override
@@ -270,7 +274,7 @@ public int mkdir(String path, @mode_t long mode) {
       // For Tencent ODFS, the groupName should be sent as null temporarily
       // Todo
       //  need to find out the reason with the policy odfs uses
-      if (mConfiguration.getBoolean(AuthConstants.AUTH_POLICY_IGNORE_MKDIR_GROUP,
+      if (mConfiguration.getBoolean(Constants.AUTH_POLICY_IGNORE_MKDIR_GROUP,
           false)) {
         mFileSystem.setOwner(turi, userName, null);
       } else {
@@ -301,6 +305,13 @@ public int open(String path, FuseFileInfo fi) {
       LOG.error("Cannot open {}: too many open files (MAX_OPEN_FILES: {})", path, MAX_OPEN_FILES);
       return ErrorCodes.EMFILE();
     }
+    while (mOpenFiles.size() >= openoncurrent) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
     FSDataInputStream is;
     FSDataOutputStream out;
     try {

diff --git a/src/main/java/hcfsfuse/fuse/HCFSJniFuseFileSystem.java b/src/main/java/hcfsfuse/fuse/HCFSJniFuseFileSystem.java
@@ -1,7 +1,7 @@
 package hcfsfuse.fuse;
 
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY;
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY_CUSTOM;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY_CUSTOM;
 
 import hcfsfuse.fuse.auth.AuthPolicy;
 import hcfsfuse.fuse.auth.AuthPolicyFactory;

diff --git a/src/main/java/hcfsfuse/fuse/auth/AuthPolicyFactory.java b/src/main/java/hcfsfuse/fuse/auth/AuthPolicyFactory.java
@@ -1,7 +1,7 @@
 package hcfsfuse.fuse.auth;
 
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY;
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY_CUSTOM;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY_CUSTOM;
 
 import alluxio.jnifuse.AbstractFuseFileSystem;
 import org.apache.hadoop.conf.Configuration;

diff --git a/src/main/java/hcfsfuse/fuse/auth/CustomAuthPolicy.java b/src/main/java/hcfsfuse/fuse/auth/CustomAuthPolicy.java
@@ -1,7 +1,7 @@
 package hcfsfuse.fuse.auth;
 
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY_CUSTOM_GROUP;
-import static hcfsfuse.fuse.AuthConstants.AUTH_POLICY_CUSTOM_USER;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY_CUSTOM_GROUP;
+import static hcfsfuse.fuse.Constants.AUTH_POLICY_CUSTOM_USER;
 
 import alluxio.jnifuse.FuseFileSystem;
 import org.apache.hadoop.conf.Configuration;