Dean.hiller/fix client guage (#232)

* track api usage per client class

* keep track of requests in flight per method and client name

* metrics can now do per api as well for in-flight stuff

---------

Co-authored-by: Dean Hiller <[email protected]>

cloud/generate-httpclient/src/main/java/org/webpieces/microsvc/client/impl/HttpsJsonClient.java

@@ -7,6 +7,7 @@
 import com.webpieces.http2.api.dto.lowlevel.lib.Http2HeaderName;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.webpieces.ctx.api.ClientServiceConfig;
@@ -44,6 +45,8 @@
 import org.webpieces.util.futures.XFuture;
 import org.webpieces.util.security.Masker;
 
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
@@ -67,7 +70,8 @@ public class HttpsJsonClient {
     private FutureHelper futureUtil;
     private final Set<String> secureList = new HashSet<>();
     private final Set<PlatformHeaders> transferHeaders = new HashSet<>();
-    private AtomicInteger counter = new AtomicInteger(0);
+
+    private final ConcurrentMap<String, AtomicInteger> clientToCounter = new ConcurrentHashMap<>();
 
     private Masker masker;
 
@@ -87,7 +91,6 @@ public HttpsJsonClient(
 
         this.serversName = clientServiceConfig.getServersName();
         this.metrics = metrics;
-        metrics.gauge("webpieces.requests.inflight.allclients", counter, (counter) -> counter.get());
 
         List<PlatformHeaders> listHeaders = clientServiceConfig.getHcl().listHeaderCtxPairs();
 
@@ -153,7 +156,6 @@ public Http2Request createHttpReq(HostWithPort apiAddress, String method, String
      */
     public <T> XFuture<T> sendHttpRequest(Method method, Object request, Endpoint endpoint, Class<T> responseType, boolean forHttp) {
 
-        String clientsName = method.getDeclaringClass().getSimpleName();
         HostWithPort apiAddress = endpoint.getServerAddress();
         String httpMethod = endpoint.getHttpMethod();
         String endpointPath = endpoint.getUrlPath();
@@ -184,7 +186,7 @@ public <T> XFuture<T> sendHttpRequest(Method method, Object request, Endpoint en
         }
 
         XFuture<T> future = futureUtil.catchBlockWrap(
-                () -> sendAndTranslate(clientsName, apiAddress, responseType, httpSocket, connect, fullRequest, jsonRequest),
+                () -> sendAndTranslate(method, apiAddress, responseType, httpSocket, connect, fullRequest, jsonRequest),
                 (t) -> translateException(httpReq, t)
         );
 
@@ -223,22 +225,48 @@ private Throwable translateException(Http2Request httpReq, Throwable t) {
 
     }
 
-    private <T> XFuture<T> sendAndTranslate(String clientsName, HostWithPort apiAddress, Class<T> responseType, Http2Socket httpSocket, XFuture<Void> connect, FullRequest fullRequest, String jsonReq) {
+    private <T> XFuture<T> sendAndTranslate(Method method, HostWithPort apiAddress, Class<T> responseType, Http2Socket httpSocket, XFuture<Void> connect, FullRequest fullRequest, String jsonReq) {
+        String clientsName = method.getDeclaringClass().getSimpleName();
+        String methodName = method.getName();
+
+        Iterable<Tag> tags = List.of(
+                Tag.of("api", clientsName),
+                Tag.of("method", methodName)
+        );
+        AtomicInteger inFlightCounter = clientToCounter.compute(clientsName, (k, v) -> computeLazyToAvoidOOM(k, v, tags));
+
         return connect
-                .thenCompose(voidd -> send(clientsName, httpSocket, fullRequest))
-                .thenApply(fullResponse -> unmarshal(clientsName, jsonReq, fullRequest, fullResponse, apiAddress.getPort(), responseType));
+                .thenCompose(voidd -> send(tags, inFlightCounter, httpSocket, fullRequest))
+                .thenApply(fullResponse -> unmarshal(jsonReq, fullRequest, fullResponse, apiAddress.getPort(), responseType));
     }
 
-    public XFuture<FullResponse> send(String clientsName, Http2Socket socket, FullRequest request) {
+    //Creating a gauage in micrometer will have micrometer thread keep a reference to object forever.
+    //It is very important to do this lazy or we add a new AtomicInteger on every api call if it
+    //already exists instead of ONCE PR API CLASS (we do not even do once per method yet, though we could)
+    public AtomicInteger computeLazyToAvoidOOM(String key, AtomicInteger existing, Iterable<Tag> tags) {
+        if(existing == null) {
+            existing = new AtomicInteger(0);
+            metrics.gauge("webpieces.requests.inflight", tags, existing, (c) -> c.get());
+        }
+        return existing;
+    }
+
+    public XFuture<FullResponse> send(Iterable<Tag> tags, AtomicInteger inFlightCounter, Http2Socket socket, FullRequest request) {
         XFuture<FullResponse> send = socket.send(request);
 
         //in an ideal world, we would use the async send which can notify us when 'write' is done so it is out the door, but
         //we never know when read is done until we parse the whole message (not exactly symmetrical)
-        counter.incrementAndGet();
-        Counter requestCounter = metrics.counter("webpieces.requests", "name", clientsName);
+        Counter requestCounter = metrics.counter("webpieces.requests", tags);
         requestCounter.increment();
+        inFlightCounter.incrementAndGet();
 
-        return send;
+        return send.thenApply( (response) -> {
+            Counter responseCounter = metrics.counter("webpieces.responses", tags);
+            responseCounter.increment();
+            inFlightCounter.decrementAndGet();
+
+            return response;
+        });
     }
 
     protected Http2Socket createSocket(HostWithPort apiAddress, Http2SocketListener listener, boolean forHttp) {
@@ -297,11 +325,7 @@ public SSLEngine createEngine(String host, int port) {
 
     }
 
-    private <T> T unmarshal(String clientsName, String jsonReq, FullRequest request, FullResponse httpResp, int port, Class<T> type) {
-
-        counter.decrementAndGet();
-        Counter responseCounter = metrics.counter("webpieces.responses", "name", clientsName);
-        responseCounter.increment();
+    private <T> T unmarshal(String jsonReq, FullRequest request, FullResponse httpResp, int port, Class<T> type) {
 
         DataWrapper payload = httpResp.getPayload();
         String contents = payload.createStringFromUtf8(0, payload.getReadableSize());

core/core-channelmanager2/src/main/java/org/webpieces/nio/api/BackpressureConfig.java

@@ -4,7 +4,7 @@
 
 public class BackpressureConfig {
 
-	private String name = "default";
+	private String name = "servers";
 
 	private boolean isLegacy = true;
 

core/core-channelmanager2/src/main/java/org/webpieces/nio/impl/cm/basic/Throttler.java

@@ -2,11 +2,15 @@
 
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
 import org.slf4j.Logger;
 import org.webpieces.nio.api.BackpressureConfig;
 import org.webpieces.nio.api.MaxRequestConfig;
 import org.webpieces.nio.api.Throttle;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class Throttler implements Throttle {
     public static final Logger LOG = Throttle.LOG;
 
@@ -23,9 +27,11 @@ public class Throttler implements Throttle {
     public Throttler(BackpressureConfig backpressureConfig, MeterRegistry metrics) {
         this.maxRequestConfig = backpressureConfig.getMaxRequestConfig();
         this.metrics = metrics;
-        incrementCounter = metrics.counter("webpieces.requests", "name", backpressureConfig.getName());
-        decrementCounter = metrics.counter("webpieces.responses", "name", backpressureConfig.getName());
-        metrics.gauge("webpieces.requests.inflight", outstandingRequests, (val) -> getValue(val));
+        List<Tag> tags = List.of(Tag.of("api", backpressureConfig.getName()));
+
+        incrementCounter = metrics.counter("webpieces.requests", tags);
+        decrementCounter = metrics.counter("webpieces.responses", tags);
+        metrics.gauge("webpieces.requests.inflight", tags, outstandingRequests, (val) -> getValue(val));
     }
 
     private synchronized double getValue(Integer count) {

webserver-plugins/plugin-hibernate/src/main/java/org/webpieces/plugin/hibernate/HibernatePlugin.java

@@ -39,6 +39,7 @@ public HibernatePlugin(HibernateConfig config) {
 		this.loadByClassFile = cmdLineArgs.createOptionalArg(LOAD_CLASSMETA_KEY, "true", "If you supply a *.class for 'hibernate.persistenceunit', set this flat to true", (s) -> convertBool(s));
 	}
 
+	@Deprecated
 	public HibernatePlugin(Arguments cmdLineArgs) {
 		log.info("classloader="+getClass().getClassLoader());
 		this.persistenceUnit = cmdLineArgs.createRequiredArg(PERSISTENCE_UNIT_KEY, null, "The named persistence unit from the list of them inside META-INF/persistence.xml", (s) -> s);

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/base/ProdServerMeta.java

@@ -85,7 +85,7 @@ public List<Plugin> getPlugins() {
 				//if you want to remove hibernate, just remove it first from the build file and then remove
 				//all the compile error code(it will remove more than half of the jar size of the web app actually due
 				//to transitive dependencies)
-				new HibernatePlugin(pluginConfig.getCmdLineArguments()),
+				new HibernatePlugin(pluginConfig.getCmdLineArguments(), false),
 				//ANY controllers in json package or subpackages are run through this filter independent of the url
 				new JacksonPlugin(new JacksonConfig().setPackageFilterPattern("webpiecesxxxxxpackage.json.*")),
 				new BackendPlugin(pluginConfig.getCmdLineArguments()),

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/web/login/AppLoginController.java

@@ -38,9 +38,7 @@ protected Action fetchGetLoginPageAction() {
 				"password", null);
 	}
 
-	public Render loginHome() {
-		return Actions.renderThis();
-	}
+
 
 	@Override
 	protected String getLoginSessionKey() {

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/web/login/LoginRoutes.java

@@ -48,9 +48,8 @@ protected String getSessionToken() {
 	}
 
 	@Override
-	protected void addLoggedInHome(RouteBuilder baseRouter, ScopedRouteBuilder scopedRouter1) {
-		ScopedRouteBuilder scopedRouter = scopedRouter1.getScopedRouteBuilder("/secure");
-		scopedRouter.addRoute(HTTPS, HttpMethod.GET ,   "/loggedinhome",        "AppLoginController.loginHome", LoginRouteId.LOGGED_IN_HOME);
+	protected void addLoggedInHome(RouteBuilder baseRouter, ScopedRouteBuilder scopedRouter) {
+		scopedRouter.addRoute(HTTPS, HttpMethod.GET ,   "/home",        "../secure/HomeController.loginHome", LoginRouteId.LOGGED_IN_HOME);
 	}
 
 	@Override

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/web/secure/HomeController.java

@@ -0,0 +1,15 @@
+package webpiecesxxxxxpackage.web.secure;
+
+import org.webpieces.router.api.controller.actions.Actions;
+import org.webpieces.router.api.controller.actions.Render;
+
+import javax.inject.Singleton;
+
+@Singleton
+public class HomeController {
+
+    public Render loginHome() {
+        return Actions.renderThis();
+    }
+
+}

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/web/secure/crud/CrudUserController.java

@@ -16,6 +16,7 @@
 import org.slf4j.LoggerFactory;
 import org.webpieces.ctx.api.Current;
 import org.webpieces.plugin.hibernate.Em;
+import org.webpieces.plugin.hibernate.Transaction;
 import org.webpieces.plugin.hibernate.UseQuery;
 import org.webpieces.router.api.controller.actions.Action;
 import org.webpieces.router.api.controller.actions.Actions;
@@ -32,15 +33,17 @@
 public class CrudUserController {
 
 	private static Logger log = LoggerFactory.getLogger(CrudUserController.class);
-
+
+	@Transaction
 	public Action userList() {
 		EntityManager mgr = Em.get();
 		Query query = mgr.createNamedQuery("findAllUsers");
 		@SuppressWarnings("unchecked")
 		List<UserDbo> users = query.getResultList();
 		return Actions.renderThis("users", users);
 	}
-
+
+	@Transaction
 	public Action userAddEdit(Integer id) {
 		if(id == null) {
 			return Actions.renderThis(
@@ -62,6 +65,7 @@ public Action userAddEdit(Integer id) {
 				"password", null);
 	}
 
+	@Transaction
 	public Redirect postSaveUser(@UseQuery("findByIdWithRoleJoin") UserDbo entity, 
 			List<RoleEnum> selectedRoles, @NotBlank @Size(min=4, max=20) String password) {
 //		Validation is done with JSR303 hibernate-validator so you don't need to do this BUT you could for fancier stuff...
@@ -114,11 +118,13 @@ public Redirect postSaveUser(@UseQuery("findByIdWithRoleJoin") UserDbo entity,
 		return Actions.redirect(CrudUserRouteId.LIST_USERS);
 	}
 
+	@Transaction
 	public Render confirmDeleteUser(int id) {
 		UserDbo user = Em.get().find(UserDbo.class, id);
 		return Actions.renderThis("entity", user);
 	}
-
+
+	@Transaction
 	public Redirect postDeleteUser(int id) {
 		UserDbo ref = Em.get().find(UserDbo.class, id);
 		List<UserRole> roles = ref.getRoles();

webserver-templates/webpiecesServerBuilder/TemplateBasic/production/src/main/java/webpiecesxxxxxpackage/web/secure/crudajax/AjaxCrudUserController.java

@@ -4,6 +4,7 @@
 
 import java.util.List;
 
+import javax.inject.Inject;
 import javax.inject.Singleton;
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
@@ -12,6 +13,8 @@
 import org.slf4j.LoggerFactory;
 import org.webpieces.ctx.api.Current;
 import org.webpieces.plugin.hibernate.Em;
+import org.webpieces.plugin.hibernate.PersistenceHelper;
+import org.webpieces.plugin.hibernate.Transaction;
 import org.webpieces.router.api.controller.actions.Action;
 import org.webpieces.router.api.controller.actions.Actions;
 import org.webpieces.router.api.controller.actions.Redirect;
@@ -24,7 +27,14 @@
 public class AjaxCrudUserController {
 
 	private static Logger log = LoggerFactory.getLogger(AjaxCrudUserController.class);
-
+	private PersistenceHelper persistence;
+
+	@Inject
+	public AjaxCrudUserController(PersistenceHelper persistence) {
+		this.persistence = persistence;
+	}
+
+	@Transaction
 	public Action userList() {
 		EntityManager mgr = Em.get();
 		Query query = mgr.createNamedQuery("findAllUsers");
@@ -35,7 +45,8 @@ public Action userList() {
 				"users", users,
 				"showPopup", showEditPopup);
 	}
-
+
+	@Transaction
 	public Action userAddEdit(Integer id) {		
 		if(id == null) {
 			return Actions.renderThis(
@@ -49,6 +60,8 @@ public Action userAddEdit(Integer id) {
 				"password", null);
 	}
 
+	@Transaction
+
 	public Redirect postSaveUser(UserDbo entity, String password) {
 		//TODO: if we wire in JSR303 bean validation into the platform, it could be 
 		//done there as well though would
@@ -90,11 +103,13 @@ public Redirect postSaveUser(UserDbo entity, String password) {
 		return Actions.redirect(AjaxCrudUserRouteId.AJAX_LIST_USERS);
 	}
 
+	@Transaction
 	public Render confirmDeleteUser(int id) {
 		UserDbo user = Em.get().find(UserDbo.class, id);
 		return Actions.renderThis("entity", user);
 	}
 
+	@Transaction
 	public Redirect postDeleteUser(int id) {
 		UserDbo ref = Em.get().find(UserDbo.class, id);
 		List<UserRole> roles = ref.getRoles();

webserver-templates/webpiecesServerBuilder/TemplateJsonMicroSvc/production/src/main/java/webpiecesxxxxxpackage/base/ProdServerMeta.java

@@ -79,7 +79,7 @@ public List<Plugin> getPlugins() {
 				//if you want to remove hibernate, just remove it first from the build file and then remove
 				//all the compile error code(it will remove more than half of the jar size of the web app actually due
 				//to transitive dependencies)
-				new HibernatePlugin(pluginConfig.getCmdLineArguments()),
+				new HibernatePlugin(pluginConfig.getCmdLineArguments(), false),
 				//ANY controllers in json package or subpackages are run through this filter independent of the url
 				new JacksonPlugin(new JacksonConfig().setPackageFilterPattern("webpiecesxxxxxpackage.json.*"))
 				);