diff --git a/READLAN.md b/READLAN.md new file mode 100644 index 0000000000000..dc95d700e5b4c --- /dev/null +++ b/READLAN.md @@ -0,0 +1 @@ +mvn install -Pcore-modules,-main -DskipTests -Dcheckstyle.skip=true diff --git a/bin/function-localrunner b/bin/function-localrunner index 45a37cb306794..c84f75861050f 100755 --- a/bin/function-localrunner +++ b/bin/function-localrunner @@ -34,7 +34,7 @@ else JAVA=$JAVA_HOME/bin/java fi -PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} +PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m -XX:+UnlockExperimentalVMOptions"} # Garbage collection options PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"} @@ -147,6 +147,8 @@ fi OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" +echo $OPTS + MAINCLASS="org.apache.pulsar.functions.LocalRunner" #Change to PULSAR_HOME to support relative paths diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh index 0747e6dacb82a..1c9e2a33f33e9 100755 --- a/build/build_java_test_image.sh +++ b/build/build_java_test_image.sh @@ -26,6 +26,6 @@ SQUASH_PARAM="" if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]; then SQUASH_PARAM="-Ddocker.squash=true" fi -mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \ +mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,docker \ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \ "$@" install \ No newline at end of file diff --git a/conf/broker.conf b/conf/broker.conf index 30e79ebc9f090..0472c202ca81a 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -24,7 +24,11 @@ # * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 # * my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 (will default to ZooKeeper when the schema is not specified) # * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181/my-chroot-path (to add a ZK chroot path) -metadataStoreUrl= +metadataStoreUrl=etcd://192.168.3.228:2379/ledgers + +# /ledgers/available +# /ledgers/cookies + # Configuration file path for metadata store. It's supported by RocksdbMetadataStore and EtcdMetadataStore for now metadataStoreConfigPath= @@ -66,9 +70,14 @@ bindAddress=0.0.0.0 # Extra bind addresses for the service: :://:,[...] bindAddresses= +<<<<<<< HEAD +# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. +advertisedAddress=192.168.3.228 +======= # Hostname or IP address the service advertises to the outside world. # If not set, the value of InetAddress.getLocalHost().getCanonicalHostName() is used. advertisedAddress= +>>>>>>> e1c0ad395c19c99369e733f44a2426de5c465787 # Used to specify multiple advertised listeners for the broker. # The value must format as :pulsar://:, @@ -119,7 +128,7 @@ enableBusyWait=false isRunningStandalone= # Name of the cluster to which this broker belongs to -clusterName= +clusterName=lan # The maximum number of tenants that each pulsar cluster can create # This configuration is not precise control, in a concurrent scenario, the threshold will be exceeded @@ -1048,13 +1057,13 @@ bookkeeperExplicitLacIntervalInMills=0 ### --- Managed Ledger --- ### # Number of bookies to use when creating a ledger -managedLedgerDefaultEnsembleSize=2 +managedLedgerDefaultEnsembleSize=1 # Number of copies to store for each message -managedLedgerDefaultWriteQuorum=2 +managedLedgerDefaultWriteQuorum=1 # Number of guaranteed copies (acks to wait before write is complete) -managedLedgerDefaultAckQuorum=2 +managedLedgerDefaultAckQuorum=1 # with OpportunisticStriping=true the ensembleSize is adapted automatically to writeQuorum # in case of lack of enough bookies @@ -1415,10 +1424,10 @@ exposeBundlesMetricsInPrometheus=false ### --- Functions --- ### # Enable Functions Worker Service in Broker -functionsWorkerEnabled=false +functionsWorkerEnabled=true #Enable Functions Worker to use packageManagement Service to store package. -functionsWorkerEnablePackageManagement=false +functionsWorkerEnablePackageManagement=true ### --- Broker Web Stats --- ### @@ -1594,10 +1603,11 @@ transactionPendingAckBatchedWriteMaxDelayInMillis=1 ### --- Packages management service configuration variables (begin) --- ### # Enable the packages management service or not -enablePackagesManagement=false +enablePackagesManagement=true # The packages management service storage service provide -packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider +#packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider +packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.filesystem.FileSystemPackagesStorageProvider # When the packages storage provider is bookkeeper, you can use this configuration to # control the number of replicas for storing the package diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 6929de9f1bea5..c0eb6c2adc300 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -132,9 +132,9 @@ functionRuntimeFactoryConfigs: # location of log files for functions logDirectory: logs/ # change the jar location only when you put the java instance jar in a different location - javaInstanceJarLocation: + javaInstanceJarLocation: conf/instances/java-instance.jar # change the python instance location only when you put the python instance jar in a different location - pythonInstanceLocation: + pythonInstanceLocation: conf/instances/python-instance # change the extra dependencies location: extraFunctionDependenciesDir: diff --git a/conf/instances/java-instance.jar b/conf/instances/java-instance.jar new file mode 100644 index 0000000000000..ce20ae96eee1e Binary files /dev/null and b/conf/instances/java-instance.jar differ diff --git a/conf/instances/python-instance/Function_pb2.py b/conf/instances/python-instance/Function_pb2.py new file mode 100644 index 0000000000000..14a33a1255d2a --- /dev/null +++ b/conf/instances/python-instance/Function_pb2.py @@ -0,0 +1,1702 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: Function.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='Function.proto', + package='proto', + syntax='proto3', + serialized_options=b'\n!org.apache.pulsar.functions.protoB\010Function', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa2\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x0f\n\x07\x61utoAck\x18\t \x01(\x08\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xe1\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb4\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xf0\x02\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*O\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3' +) + +_PROCESSINGGUARANTEES = _descriptor.EnumDescriptor( + name='ProcessingGuarantees', + full_name='proto.ProcessingGuarantees', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='ATLEAST_ONCE', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ATMOST_ONCE', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EFFECTIVELY_ONCE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=3685, + serialized_end=3764, +) +_sym_db.RegisterEnumDescriptor(_PROCESSINGGUARANTEES) + +ProcessingGuarantees = enum_type_wrapper.EnumTypeWrapper(_PROCESSINGGUARANTEES) +_SUBSCRIPTIONTYPE = _descriptor.EnumDescriptor( + name='SubscriptionType', + full_name='proto.SubscriptionType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='SHARED', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='FAILOVER', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='KEY_SHARED', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=3766, + serialized_end=3826, +) +_sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONTYPE) + +SubscriptionType = enum_type_wrapper.EnumTypeWrapper(_SUBSCRIPTIONTYPE) +_SUBSCRIPTIONPOSITION = _descriptor.EnumDescriptor( + name='SubscriptionPosition', + full_name='proto.SubscriptionPosition', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='LATEST', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EARLIEST', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=3828, + serialized_end=3876, +) +_sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONPOSITION) + +SubscriptionPosition = enum_type_wrapper.EnumTypeWrapper(_SUBSCRIPTIONPOSITION) +_FUNCTIONSTATE = _descriptor.EnumDescriptor( + name='FunctionState', + full_name='proto.FunctionState', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='RUNNING', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STOPPED', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=3878, + serialized_end=3919, +) +_sym_db.RegisterEnumDescriptor(_FUNCTIONSTATE) + +FunctionState = enum_type_wrapper.EnumTypeWrapper(_FUNCTIONSTATE) +ATLEAST_ONCE = 0 +ATMOST_ONCE = 1 +EFFECTIVELY_ONCE = 2 +SHARED = 0 +FAILOVER = 1 +KEY_SHARED = 2 +LATEST = 0 +EARLIEST = 1 +RUNNING = 0 +STOPPED = 1 + + +_FUNCTIONDETAILS_RUNTIME = _descriptor.EnumDescriptor( + name='Runtime', + full_name='proto.FunctionDetails.Runtime', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='JAVA', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='PYTHON', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='GO', index=2, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=844, + serialized_end=883, +) +_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_RUNTIME) + +_FUNCTIONDETAILS_COMPONENTTYPE = _descriptor.EnumDescriptor( + name='ComponentType', + full_name='proto.FunctionDetails.ComponentType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='FUNCTION', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SOURCE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SINK', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=885, + serialized_end=949, +) +_sym_db.RegisterEnumDescriptor(_FUNCTIONDETAILS_COMPONENTTYPE) + +_CRYPTOSPEC_FAILUREACTION = _descriptor.EnumDescriptor( + name='FailureAction', + full_name='proto.CryptoSpec.FailureAction', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='FAIL', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='DISCARD', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='CONSUME', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SEND', index=3, number=10, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=1873, + serialized_end=1934, +) +_sym_db.RegisterEnumDescriptor(_CRYPTOSPEC_FAILUREACTION) + + +_RESOURCES = _descriptor.Descriptor( + name='Resources', + full_name='proto.Resources', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='cpu', full_name='proto.Resources.cpu', index=0, + number=1, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='ram', full_name='proto.Resources.ram', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='disk', full_name='proto.Resources.disk', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=25, + serialized_end=76, +) + + +_RETRYDETAILS = _descriptor.Descriptor( + name='RetryDetails', + full_name='proto.RetryDetails', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='maxMessageRetries', full_name='proto.RetryDetails.maxMessageRetries', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deadLetterTopic', full_name='proto.RetryDetails.deadLetterTopic', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=78, + serialized_end=144, +) + + +_FUNCTIONDETAILS = _descriptor.Descriptor( + name='FunctionDetails', + full_name='proto.FunctionDetails', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='tenant', full_name='proto.FunctionDetails.tenant', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='namespace', full_name='proto.FunctionDetails.namespace', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='name', full_name='proto.FunctionDetails.name', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='className', full_name='proto.FunctionDetails.className', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='logTopic', full_name='proto.FunctionDetails.logTopic', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='processingGuarantees', full_name='proto.FunctionDetails.processingGuarantees', index=5, + number=6, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='userConfig', full_name='proto.FunctionDetails.userConfig', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='secretsMap', full_name='proto.FunctionDetails.secretsMap', index=7, + number=16, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='runtime', full_name='proto.FunctionDetails.runtime', index=8, + number=8, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='autoAck', full_name='proto.FunctionDetails.autoAck', index=9, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='parallelism', full_name='proto.FunctionDetails.parallelism', index=10, + number=10, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='source', full_name='proto.FunctionDetails.source', index=11, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='sink', full_name='proto.FunctionDetails.sink', index=12, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='resources', full_name='proto.FunctionDetails.resources', index=13, + number=13, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='packageUrl', full_name='proto.FunctionDetails.packageUrl', index=14, + number=14, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='retryDetails', full_name='proto.FunctionDetails.retryDetails', index=15, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='runtimeFlags', full_name='proto.FunctionDetails.runtimeFlags', index=16, + number=17, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='componentType', full_name='proto.FunctionDetails.componentType', index=17, + number=18, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='customRuntimeOptions', full_name='proto.FunctionDetails.customRuntimeOptions', index=18, + number=19, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='builtin', full_name='proto.FunctionDetails.builtin', index=19, + number=20, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='retainOrdering', full_name='proto.FunctionDetails.retainOrdering', index=20, + number=21, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='retainKeyOrdering', full_name='proto.FunctionDetails.retainKeyOrdering', index=21, + number=22, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='subscriptionPosition', full_name='proto.FunctionDetails.subscriptionPosition', index=22, + number=23, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _FUNCTIONDETAILS_RUNTIME, + _FUNCTIONDETAILS_COMPONENTTYPE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=147, + serialized_end=949, +) + + +_CONSUMERSPEC_RECEIVERQUEUESIZE = _descriptor.Descriptor( + name='ReceiverQueueSize', + full_name='proto.ConsumerSpec.ReceiverQueueSize', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='proto.ConsumerSpec.ReceiverQueueSize.value', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1283, + serialized_end=1317, +) + +_CONSUMERSPEC_SCHEMAPROPERTIESENTRY = _descriptor.Descriptor( + name='SchemaPropertiesEntry', + full_name='proto.ConsumerSpec.SchemaPropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.ConsumerSpec.SchemaPropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.ConsumerSpec.SchemaPropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1319, + serialized_end=1374, +) + +_CONSUMERSPEC_CONSUMERPROPERTIESENTRY = _descriptor.Descriptor( + name='ConsumerPropertiesEntry', + full_name='proto.ConsumerSpec.ConsumerPropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.ConsumerSpec.ConsumerPropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.ConsumerSpec.ConsumerPropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1376, + serialized_end=1433, +) + +_CONSUMERSPEC = _descriptor.Descriptor( + name='ConsumerSpec', + full_name='proto.ConsumerSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='schemaType', full_name='proto.ConsumerSpec.schemaType', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='serdeClassName', full_name='proto.ConsumerSpec.serdeClassName', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='isRegexPattern', full_name='proto.ConsumerSpec.isRegexPattern', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='receiverQueueSize', full_name='proto.ConsumerSpec.receiverQueueSize', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='schemaProperties', full_name='proto.ConsumerSpec.schemaProperties', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='consumerProperties', full_name='proto.ConsumerSpec.consumerProperties', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cryptoSpec', full_name='proto.ConsumerSpec.cryptoSpec', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_CONSUMERSPEC_RECEIVERQUEUESIZE, _CONSUMERSPEC_SCHEMAPROPERTIESENTRY, _CONSUMERSPEC_CONSUMERPROPERTIESENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=952, + serialized_end=1433, +) + + +_PRODUCERSPEC = _descriptor.Descriptor( + name='ProducerSpec', + full_name='proto.ProducerSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='maxPendingMessages', full_name='proto.ProducerSpec.maxPendingMessages', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='maxPendingMessagesAcrossPartitions', full_name='proto.ProducerSpec.maxPendingMessagesAcrossPartitions', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='useThreadLocalProducers', full_name='proto.ProducerSpec.useThreadLocalProducers', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cryptoSpec', full_name='proto.ProducerSpec.cryptoSpec', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='batchBuilder', full_name='proto.ProducerSpec.batchBuilder', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1436, + serialized_end=1616, +) + + +_CRYPTOSPEC = _descriptor.Descriptor( + name='CryptoSpec', + full_name='proto.CryptoSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='cryptoKeyReaderClassName', full_name='proto.CryptoSpec.cryptoKeyReaderClassName', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cryptoKeyReaderConfig', full_name='proto.CryptoSpec.cryptoKeyReaderConfig', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='producerEncryptionKeyName', full_name='proto.CryptoSpec.producerEncryptionKeyName', index=2, + number=3, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='producerCryptoFailureAction', full_name='proto.CryptoSpec.producerCryptoFailureAction', index=3, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='consumerCryptoFailureAction', full_name='proto.CryptoSpec.consumerCryptoFailureAction', index=4, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _CRYPTOSPEC_FAILUREACTION, + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1619, + serialized_end=1934, +) + + +_SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY = _descriptor.Descriptor( + name='TopicsToSerDeClassNameEntry', + full_name='proto.SourceSpec.TopicsToSerDeClassNameEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.SourceSpec.TopicsToSerDeClassNameEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.SourceSpec.TopicsToSerDeClassNameEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2414, + serialized_end=2475, +) + +_SOURCESPEC_INPUTSPECSENTRY = _descriptor.Descriptor( + name='InputSpecsEntry', + full_name='proto.SourceSpec.InputSpecsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.SourceSpec.InputSpecsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.SourceSpec.InputSpecsEntry.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2477, + serialized_end=2547, +) + +_SOURCESPEC = _descriptor.Descriptor( + name='SourceSpec', + full_name='proto.SourceSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='className', full_name='proto.SourceSpec.className', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='configs', full_name='proto.SourceSpec.configs', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='typeClassName', full_name='proto.SourceSpec.typeClassName', index=2, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='subscriptionType', full_name='proto.SourceSpec.subscriptionType', index=3, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='topicsToSerDeClassName', full_name='proto.SourceSpec.topicsToSerDeClassName', index=4, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='inputSpecs', full_name='proto.SourceSpec.inputSpecs', index=5, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='timeoutMs', full_name='proto.SourceSpec.timeoutMs', index=6, + number=6, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='topicsPattern', full_name='proto.SourceSpec.topicsPattern', index=7, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='builtin', full_name='proto.SourceSpec.builtin', index=8, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='subscriptionName', full_name='proto.SourceSpec.subscriptionName', index=9, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cleanupSubscription', full_name='proto.SourceSpec.cleanupSubscription', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='subscriptionPosition', full_name='proto.SourceSpec.subscriptionPosition', index=11, + number=12, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='negativeAckRedeliveryDelayMs', full_name='proto.SourceSpec.negativeAckRedeliveryDelayMs', index=12, + number=13, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY, _SOURCESPEC_INPUTSPECSENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1937, + serialized_end=2547, +) + + +_SINKSPEC_SCHEMAPROPERTIESENTRY = _descriptor.Descriptor( + name='SchemaPropertiesEntry', + full_name='proto.SinkSpec.SchemaPropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.SinkSpec.SchemaPropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.SinkSpec.SchemaPropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1319, + serialized_end=1374, +) + +_SINKSPEC_CONSUMERPROPERTIESENTRY = _descriptor.Descriptor( + name='ConsumerPropertiesEntry', + full_name='proto.SinkSpec.ConsumerPropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.SinkSpec.ConsumerPropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.SinkSpec.ConsumerPropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1376, + serialized_end=1433, +) + +_SINKSPEC = _descriptor.Descriptor( + name='SinkSpec', + full_name='proto.SinkSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='className', full_name='proto.SinkSpec.className', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='configs', full_name='proto.SinkSpec.configs', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='typeClassName', full_name='proto.SinkSpec.typeClassName', index=2, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='topic', full_name='proto.SinkSpec.topic', index=3, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='producerSpec', full_name='proto.SinkSpec.producerSpec', index=4, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='serDeClassName', full_name='proto.SinkSpec.serDeClassName', index=5, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='builtin', full_name='proto.SinkSpec.builtin', index=6, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='schemaType', full_name='proto.SinkSpec.schemaType', index=7, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='forwardSourceMessageProperty', full_name='proto.SinkSpec.forwardSourceMessageProperty', index=8, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='schemaProperties', full_name='proto.SinkSpec.schemaProperties', index=9, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='consumerProperties', full_name='proto.SinkSpec.consumerProperties', index=10, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_SINKSPEC_SCHEMAPROPERTIESENTRY, _SINKSPEC_CONSUMERPROPERTIESENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2550, + serialized_end=3026, +) + + +_PACKAGELOCATIONMETADATA = _descriptor.Descriptor( + name='PackageLocationMetaData', + full_name='proto.PackageLocationMetaData', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='packagePath', full_name='proto.PackageLocationMetaData.packagePath', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='originalFileName', full_name='proto.PackageLocationMetaData.originalFileName', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3028, + serialized_end=3100, +) + + +_FUNCTIONMETADATA_INSTANCESTATESENTRY = _descriptor.Descriptor( + name='InstanceStatesEntry', + full_name='proto.FunctionMetaData.InstanceStatesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.FunctionMetaData.InstanceStatesEntry.key', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.FunctionMetaData.InstanceStatesEntry.value', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3396, + serialized_end=3471, +) + +_FUNCTIONMETADATA = _descriptor.Descriptor( + name='FunctionMetaData', + full_name='proto.FunctionMetaData', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='functionDetails', full_name='proto.FunctionMetaData.functionDetails', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='packageLocation', full_name='proto.FunctionMetaData.packageLocation', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='version', full_name='proto.FunctionMetaData.version', index=2, + number=3, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='createTime', full_name='proto.FunctionMetaData.createTime', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='instanceStates', full_name='proto.FunctionMetaData.instanceStates', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='functionAuthSpec', full_name='proto.FunctionMetaData.functionAuthSpec', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_FUNCTIONMETADATA_INSTANCESTATESENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3103, + serialized_end=3471, +) + + +_FUNCTIONAUTHENTICATIONSPEC = _descriptor.Descriptor( + name='FunctionAuthenticationSpec', + full_name='proto.FunctionAuthenticationSpec', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='data', full_name='proto.FunctionAuthenticationSpec.data', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='provider', full_name='proto.FunctionAuthenticationSpec.provider', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3473, + serialized_end=3533, +) + + +_INSTANCE = _descriptor.Descriptor( + name='Instance', + full_name='proto.Instance', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='functionMetaData', full_name='proto.Instance.functionMetaData', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='instanceId', full_name='proto.Instance.instanceId', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3535, + serialized_end=3616, +) + + +_ASSIGNMENT = _descriptor.Descriptor( + name='Assignment', + full_name='proto.Assignment', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='instance', full_name='proto.Assignment.instance', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='workerId', full_name='proto.Assignment.workerId', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=3618, + serialized_end=3683, +) + +_FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _PROCESSINGGUARANTEES +_FUNCTIONDETAILS.fields_by_name['runtime'].enum_type = _FUNCTIONDETAILS_RUNTIME +_FUNCTIONDETAILS.fields_by_name['source'].message_type = _SOURCESPEC +_FUNCTIONDETAILS.fields_by_name['sink'].message_type = _SINKSPEC +_FUNCTIONDETAILS.fields_by_name['resources'].message_type = _RESOURCES +_FUNCTIONDETAILS.fields_by_name['retryDetails'].message_type = _RETRYDETAILS +_FUNCTIONDETAILS.fields_by_name['componentType'].enum_type = _FUNCTIONDETAILS_COMPONENTTYPE +_FUNCTIONDETAILS.fields_by_name['subscriptionPosition'].enum_type = _SUBSCRIPTIONPOSITION +_FUNCTIONDETAILS_RUNTIME.containing_type = _FUNCTIONDETAILS +_FUNCTIONDETAILS_COMPONENTTYPE.containing_type = _FUNCTIONDETAILS +_CONSUMERSPEC_RECEIVERQUEUESIZE.containing_type = _CONSUMERSPEC +_CONSUMERSPEC_SCHEMAPROPERTIESENTRY.containing_type = _CONSUMERSPEC +_CONSUMERSPEC_CONSUMERPROPERTIESENTRY.containing_type = _CONSUMERSPEC +_CONSUMERSPEC.fields_by_name['receiverQueueSize'].message_type = _CONSUMERSPEC_RECEIVERQUEUESIZE +_CONSUMERSPEC.fields_by_name['schemaProperties'].message_type = _CONSUMERSPEC_SCHEMAPROPERTIESENTRY +_CONSUMERSPEC.fields_by_name['consumerProperties'].message_type = _CONSUMERSPEC_CONSUMERPROPERTIESENTRY +_CONSUMERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC +_PRODUCERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC +_CRYPTOSPEC.fields_by_name['producerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION +_CRYPTOSPEC.fields_by_name['consumerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION +_CRYPTOSPEC_FAILUREACTION.containing_type = _CRYPTOSPEC +_SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY.containing_type = _SOURCESPEC +_SOURCESPEC_INPUTSPECSENTRY.fields_by_name['value'].message_type = _CONSUMERSPEC +_SOURCESPEC_INPUTSPECSENTRY.containing_type = _SOURCESPEC +_SOURCESPEC.fields_by_name['subscriptionType'].enum_type = _SUBSCRIPTIONTYPE +_SOURCESPEC.fields_by_name['topicsToSerDeClassName'].message_type = _SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY +_SOURCESPEC.fields_by_name['inputSpecs'].message_type = _SOURCESPEC_INPUTSPECSENTRY +_SOURCESPEC.fields_by_name['subscriptionPosition'].enum_type = _SUBSCRIPTIONPOSITION +_SINKSPEC_SCHEMAPROPERTIESENTRY.containing_type = _SINKSPEC +_SINKSPEC_CONSUMERPROPERTIESENTRY.containing_type = _SINKSPEC +_SINKSPEC.fields_by_name['producerSpec'].message_type = _PRODUCERSPEC +_SINKSPEC.fields_by_name['schemaProperties'].message_type = _SINKSPEC_SCHEMAPROPERTIESENTRY +_SINKSPEC.fields_by_name['consumerProperties'].message_type = _SINKSPEC_CONSUMERPROPERTIESENTRY +_FUNCTIONMETADATA_INSTANCESTATESENTRY.fields_by_name['value'].enum_type = _FUNCTIONSTATE +_FUNCTIONMETADATA_INSTANCESTATESENTRY.containing_type = _FUNCTIONMETADATA +_FUNCTIONMETADATA.fields_by_name['functionDetails'].message_type = _FUNCTIONDETAILS +_FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA +_FUNCTIONMETADATA.fields_by_name['instanceStates'].message_type = _FUNCTIONMETADATA_INSTANCESTATESENTRY +_FUNCTIONMETADATA.fields_by_name['functionAuthSpec'].message_type = _FUNCTIONAUTHENTICATIONSPEC +_INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA +_ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE +DESCRIPTOR.message_types_by_name['Resources'] = _RESOURCES +DESCRIPTOR.message_types_by_name['RetryDetails'] = _RETRYDETAILS +DESCRIPTOR.message_types_by_name['FunctionDetails'] = _FUNCTIONDETAILS +DESCRIPTOR.message_types_by_name['ConsumerSpec'] = _CONSUMERSPEC +DESCRIPTOR.message_types_by_name['ProducerSpec'] = _PRODUCERSPEC +DESCRIPTOR.message_types_by_name['CryptoSpec'] = _CRYPTOSPEC +DESCRIPTOR.message_types_by_name['SourceSpec'] = _SOURCESPEC +DESCRIPTOR.message_types_by_name['SinkSpec'] = _SINKSPEC +DESCRIPTOR.message_types_by_name['PackageLocationMetaData'] = _PACKAGELOCATIONMETADATA +DESCRIPTOR.message_types_by_name['FunctionMetaData'] = _FUNCTIONMETADATA +DESCRIPTOR.message_types_by_name['FunctionAuthenticationSpec'] = _FUNCTIONAUTHENTICATIONSPEC +DESCRIPTOR.message_types_by_name['Instance'] = _INSTANCE +DESCRIPTOR.message_types_by_name['Assignment'] = _ASSIGNMENT +DESCRIPTOR.enum_types_by_name['ProcessingGuarantees'] = _PROCESSINGGUARANTEES +DESCRIPTOR.enum_types_by_name['SubscriptionType'] = _SUBSCRIPTIONTYPE +DESCRIPTOR.enum_types_by_name['SubscriptionPosition'] = _SUBSCRIPTIONPOSITION +DESCRIPTOR.enum_types_by_name['FunctionState'] = _FUNCTIONSTATE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Resources = _reflection.GeneratedProtocolMessageType('Resources', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCES, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.Resources) + }) +_sym_db.RegisterMessage(Resources) + +RetryDetails = _reflection.GeneratedProtocolMessageType('RetryDetails', (_message.Message,), { + 'DESCRIPTOR' : _RETRYDETAILS, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.RetryDetails) + }) +_sym_db.RegisterMessage(RetryDetails) + +FunctionDetails = _reflection.GeneratedProtocolMessageType('FunctionDetails', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTIONDETAILS, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionDetails) + }) +_sym_db.RegisterMessage(FunctionDetails) + +ConsumerSpec = _reflection.GeneratedProtocolMessageType('ConsumerSpec', (_message.Message,), { + + 'ReceiverQueueSize' : _reflection.GeneratedProtocolMessageType('ReceiverQueueSize', (_message.Message,), { + 'DESCRIPTOR' : _CONSUMERSPEC_RECEIVERQUEUESIZE, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.ConsumerSpec.ReceiverQueueSize) + }) + , + + 'SchemaPropertiesEntry' : _reflection.GeneratedProtocolMessageType('SchemaPropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _CONSUMERSPEC_SCHEMAPROPERTIESENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.ConsumerSpec.SchemaPropertiesEntry) + }) + , + + 'ConsumerPropertiesEntry' : _reflection.GeneratedProtocolMessageType('ConsumerPropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _CONSUMERSPEC_CONSUMERPROPERTIESENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.ConsumerSpec.ConsumerPropertiesEntry) + }) + , + 'DESCRIPTOR' : _CONSUMERSPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.ConsumerSpec) + }) +_sym_db.RegisterMessage(ConsumerSpec) +_sym_db.RegisterMessage(ConsumerSpec.ReceiverQueueSize) +_sym_db.RegisterMessage(ConsumerSpec.SchemaPropertiesEntry) +_sym_db.RegisterMessage(ConsumerSpec.ConsumerPropertiesEntry) + +ProducerSpec = _reflection.GeneratedProtocolMessageType('ProducerSpec', (_message.Message,), { + 'DESCRIPTOR' : _PRODUCERSPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.ProducerSpec) + }) +_sym_db.RegisterMessage(ProducerSpec) + +CryptoSpec = _reflection.GeneratedProtocolMessageType('CryptoSpec', (_message.Message,), { + 'DESCRIPTOR' : _CRYPTOSPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.CryptoSpec) + }) +_sym_db.RegisterMessage(CryptoSpec) + +SourceSpec = _reflection.GeneratedProtocolMessageType('SourceSpec', (_message.Message,), { + + 'TopicsToSerDeClassNameEntry' : _reflection.GeneratedProtocolMessageType('TopicsToSerDeClassNameEntry', (_message.Message,), { + 'DESCRIPTOR' : _SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SourceSpec.TopicsToSerDeClassNameEntry) + }) + , + + 'InputSpecsEntry' : _reflection.GeneratedProtocolMessageType('InputSpecsEntry', (_message.Message,), { + 'DESCRIPTOR' : _SOURCESPEC_INPUTSPECSENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SourceSpec.InputSpecsEntry) + }) + , + 'DESCRIPTOR' : _SOURCESPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SourceSpec) + }) +_sym_db.RegisterMessage(SourceSpec) +_sym_db.RegisterMessage(SourceSpec.TopicsToSerDeClassNameEntry) +_sym_db.RegisterMessage(SourceSpec.InputSpecsEntry) + +SinkSpec = _reflection.GeneratedProtocolMessageType('SinkSpec', (_message.Message,), { + + 'SchemaPropertiesEntry' : _reflection.GeneratedProtocolMessageType('SchemaPropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _SINKSPEC_SCHEMAPROPERTIESENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SinkSpec.SchemaPropertiesEntry) + }) + , + + 'ConsumerPropertiesEntry' : _reflection.GeneratedProtocolMessageType('ConsumerPropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _SINKSPEC_CONSUMERPROPERTIESENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SinkSpec.ConsumerPropertiesEntry) + }) + , + 'DESCRIPTOR' : _SINKSPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.SinkSpec) + }) +_sym_db.RegisterMessage(SinkSpec) +_sym_db.RegisterMessage(SinkSpec.SchemaPropertiesEntry) +_sym_db.RegisterMessage(SinkSpec.ConsumerPropertiesEntry) + +PackageLocationMetaData = _reflection.GeneratedProtocolMessageType('PackageLocationMetaData', (_message.Message,), { + 'DESCRIPTOR' : _PACKAGELOCATIONMETADATA, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.PackageLocationMetaData) + }) +_sym_db.RegisterMessage(PackageLocationMetaData) + +FunctionMetaData = _reflection.GeneratedProtocolMessageType('FunctionMetaData', (_message.Message,), { + + 'InstanceStatesEntry' : _reflection.GeneratedProtocolMessageType('InstanceStatesEntry', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTIONMETADATA_INSTANCESTATESENTRY, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionMetaData.InstanceStatesEntry) + }) + , + 'DESCRIPTOR' : _FUNCTIONMETADATA, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionMetaData) + }) +_sym_db.RegisterMessage(FunctionMetaData) +_sym_db.RegisterMessage(FunctionMetaData.InstanceStatesEntry) + +FunctionAuthenticationSpec = _reflection.GeneratedProtocolMessageType('FunctionAuthenticationSpec', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTIONAUTHENTICATIONSPEC, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionAuthenticationSpec) + }) +_sym_db.RegisterMessage(FunctionAuthenticationSpec) + +Instance = _reflection.GeneratedProtocolMessageType('Instance', (_message.Message,), { + 'DESCRIPTOR' : _INSTANCE, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.Instance) + }) +_sym_db.RegisterMessage(Instance) + +Assignment = _reflection.GeneratedProtocolMessageType('Assignment', (_message.Message,), { + 'DESCRIPTOR' : _ASSIGNMENT, + '__module__' : 'Function_pb2' + # @@protoc_insertion_point(class_scope:proto.Assignment) + }) +_sym_db.RegisterMessage(Assignment) + + +DESCRIPTOR._options = None +_CONSUMERSPEC_SCHEMAPROPERTIESENTRY._options = None +_CONSUMERSPEC_CONSUMERPROPERTIESENTRY._options = None +_SOURCESPEC_TOPICSTOSERDECLASSNAMEENTRY._options = None +_SOURCESPEC_INPUTSPECSENTRY._options = None +_SOURCESPEC.fields_by_name['topicsToSerDeClassName']._options = None +_SOURCESPEC.fields_by_name['topicsPattern']._options = None +_SINKSPEC_SCHEMAPROPERTIESENTRY._options = None +_SINKSPEC_CONSUMERPROPERTIESENTRY._options = None +_FUNCTIONMETADATA_INSTANCESTATESENTRY._options = None +# @@protoc_insertion_point(module_scope) diff --git a/conf/instances/python-instance/InstanceCommunication_pb2.py b/conf/instances/python-instance/InstanceCommunication_pb2.py new file mode 100644 index 0000000000000..d1ba3bb0ed094 --- /dev/null +++ b/conf/instances/python-instance/InstanceCommunication_pb2.py @@ -0,0 +1,670 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: InstanceCommunication.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='InstanceCommunication.proto', + package='proto', + syntax='proto3', + serialized_options=b'\n!org.apache.pulsar.functions.protoB\025InstanceCommunication', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x1bInstanceCommunication.proto\x12\x05proto\x1a\x1bgoogle/protobuf/empty.proto\"\xc4\x05\n\x0e\x46unctionStatus\x12\x0f\n\x07running\x18\x01 \x01(\x08\x12\x18\n\x10\x66\x61ilureException\x18\x02 \x01(\t\x12\x13\n\x0bnumRestarts\x18\x03 \x01(\x03\x12\x13\n\x0bnumReceived\x18\x11 \x01(\x03\x12 \n\x18numSuccessfullyProcessed\x18\x05 \x01(\x03\x12\x19\n\x11numUserExceptions\x18\x06 \x01(\x03\x12H\n\x14latestUserExceptions\x18\x07 \x03(\x0b\x32*.proto.FunctionStatus.ExceptionInformation\x12\x1b\n\x13numSystemExceptions\x18\x08 \x01(\x03\x12J\n\x16latestSystemExceptions\x18\t \x03(\x0b\x32*.proto.FunctionStatus.ExceptionInformation\x12\x1b\n\x13numSourceExceptions\x18\x12 \x01(\x03\x12J\n\x16latestSourceExceptions\x18\x13 \x03(\x0b\x32*.proto.FunctionStatus.ExceptionInformation\x12\x19\n\x11numSinkExceptions\x18\x14 \x01(\x03\x12H\n\x14latestSinkExceptions\x18\x15 \x03(\x0b\x32*.proto.FunctionStatus.ExceptionInformation\x12\x16\n\x0e\x61verageLatency\x18\x0c \x01(\x01\x12\x1a\n\x12lastInvocationTime\x18\r \x01(\x03\x12\x12\n\ninstanceId\x18\x0e \x01(\t\x12\x10\n\x08workerId\x18\x10 \x01(\t\x1a\x45\n\x14\x45xceptionInformation\x12\x17\n\x0f\x65xceptionString\x18\x01 \x01(\t\x12\x14\n\x0cmsSinceEpoch\x18\x02 \x01(\x03\"V\n\x12\x46unctionStatusList\x12\r\n\x05\x65rror\x18\x02 \x01(\t\x12\x31\n\x12\x66unctionStatusList\x18\x01 \x03(\x0b\x32\x15.proto.FunctionStatus\"\xd0\x03\n\x0bMetricsData\x12\x15\n\rreceivedTotal\x18\x02 \x01(\x03\x12\x1a\n\x12receivedTotal_1min\x18\n \x01(\x03\x12\"\n\x1aprocessedSuccessfullyTotal\x18\x04 \x01(\x03\x12\'\n\x1fprocessedSuccessfullyTotal_1min\x18\x0c \x01(\x03\x12\x1d\n\x15systemExceptionsTotal\x18\x05 \x01(\x03\x12\"\n\x1asystemExceptionsTotal_1min\x18\r \x01(\x03\x12\x1b\n\x13userExceptionsTotal\x18\x06 \x01(\x03\x12 \n\x18userExceptionsTotal_1min\x18\x0e \x01(\x03\x12\x19\n\x11\x61vgProcessLatency\x18\x07 \x01(\x01\x12\x1e\n\x16\x61vgProcessLatency_1min\x18\x0f \x01(\x01\x12\x16\n\x0elastInvocation\x18\x08 \x01(\x03\x12\x38\n\x0buserMetrics\x18\t \x03(\x0b\x32#.proto.MetricsData.UserMetricsEntry\x1a\x32\n\x10UserMetricsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"$\n\x11HealthCheckResult\x12\x0f\n\x07success\x18\x01 \x01(\x08\"\x98\x01\n\x07Metrics\x12/\n\x07metrics\x18\x01 \x03(\x0b\x32\x1e.proto.Metrics.InstanceMetrics\x1a\\\n\x0fInstanceMetrics\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\x12\'\n\x0bmetricsData\x18\x03 \x01(\x0b\x32\x12.proto.MetricsData2\xdc\x02\n\x0fInstanceControl\x12\x44\n\x11GetFunctionStatus\x12\x16.google.protobuf.Empty\x1a\x15.proto.FunctionStatus\"\x00\x12\x42\n\x12GetAndResetMetrics\x12\x16.google.protobuf.Empty\x1a\x12.proto.MetricsData\"\x00\x12@\n\x0cResetMetrics\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12:\n\nGetMetrics\x12\x16.google.protobuf.Empty\x1a\x12.proto.MetricsData\"\x00\x12\x41\n\x0bHealthCheck\x12\x16.google.protobuf.Empty\x1a\x18.proto.HealthCheckResult\"\x00\x42:\n!org.apache.pulsar.functions.protoB\x15InstanceCommunicationb\x06proto3' + , + dependencies=[google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) + + + + +_FUNCTIONSTATUS_EXCEPTIONINFORMATION = _descriptor.Descriptor( + name='ExceptionInformation', + full_name='proto.FunctionStatus.ExceptionInformation', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='exceptionString', full_name='proto.FunctionStatus.ExceptionInformation.exceptionString', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='msSinceEpoch', full_name='proto.FunctionStatus.ExceptionInformation.msSinceEpoch', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=707, + serialized_end=776, +) + +_FUNCTIONSTATUS = _descriptor.Descriptor( + name='FunctionStatus', + full_name='proto.FunctionStatus', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='running', full_name='proto.FunctionStatus.running', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='failureException', full_name='proto.FunctionStatus.failureException', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numRestarts', full_name='proto.FunctionStatus.numRestarts', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numReceived', full_name='proto.FunctionStatus.numReceived', index=3, + number=17, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numSuccessfullyProcessed', full_name='proto.FunctionStatus.numSuccessfullyProcessed', index=4, + number=5, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numUserExceptions', full_name='proto.FunctionStatus.numUserExceptions', index=5, + number=6, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='latestUserExceptions', full_name='proto.FunctionStatus.latestUserExceptions', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numSystemExceptions', full_name='proto.FunctionStatus.numSystemExceptions', index=7, + number=8, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='latestSystemExceptions', full_name='proto.FunctionStatus.latestSystemExceptions', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numSourceExceptions', full_name='proto.FunctionStatus.numSourceExceptions', index=9, + number=18, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='latestSourceExceptions', full_name='proto.FunctionStatus.latestSourceExceptions', index=10, + number=19, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='numSinkExceptions', full_name='proto.FunctionStatus.numSinkExceptions', index=11, + number=20, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='latestSinkExceptions', full_name='proto.FunctionStatus.latestSinkExceptions', index=12, + number=21, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='averageLatency', full_name='proto.FunctionStatus.averageLatency', index=13, + number=12, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='lastInvocationTime', full_name='proto.FunctionStatus.lastInvocationTime', index=14, + number=13, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='instanceId', full_name='proto.FunctionStatus.instanceId', index=15, + number=14, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='workerId', full_name='proto.FunctionStatus.workerId', index=16, + number=16, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_FUNCTIONSTATUS_EXCEPTIONINFORMATION, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=68, + serialized_end=776, +) + + +_FUNCTIONSTATUSLIST = _descriptor.Descriptor( + name='FunctionStatusList', + full_name='proto.FunctionStatusList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='error', full_name='proto.FunctionStatusList.error', index=0, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='functionStatusList', full_name='proto.FunctionStatusList.functionStatusList', index=1, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=778, + serialized_end=864, +) + + +_METRICSDATA_USERMETRICSENTRY = _descriptor.Descriptor( + name='UserMetricsEntry', + full_name='proto.MetricsData.UserMetricsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='proto.MetricsData.UserMetricsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='proto.MetricsData.UserMetricsEntry.value', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1281, + serialized_end=1331, +) + +_METRICSDATA = _descriptor.Descriptor( + name='MetricsData', + full_name='proto.MetricsData', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='receivedTotal', full_name='proto.MetricsData.receivedTotal', index=0, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='receivedTotal_1min', full_name='proto.MetricsData.receivedTotal_1min', index=1, + number=10, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='processedSuccessfullyTotal', full_name='proto.MetricsData.processedSuccessfullyTotal', index=2, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='processedSuccessfullyTotal_1min', full_name='proto.MetricsData.processedSuccessfullyTotal_1min', index=3, + number=12, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='systemExceptionsTotal', full_name='proto.MetricsData.systemExceptionsTotal', index=4, + number=5, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='systemExceptionsTotal_1min', full_name='proto.MetricsData.systemExceptionsTotal_1min', index=5, + number=13, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='userExceptionsTotal', full_name='proto.MetricsData.userExceptionsTotal', index=6, + number=6, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='userExceptionsTotal_1min', full_name='proto.MetricsData.userExceptionsTotal_1min', index=7, + number=14, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='avgProcessLatency', full_name='proto.MetricsData.avgProcessLatency', index=8, + number=7, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='avgProcessLatency_1min', full_name='proto.MetricsData.avgProcessLatency_1min', index=9, + number=15, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='lastInvocation', full_name='proto.MetricsData.lastInvocation', index=10, + number=8, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='userMetrics', full_name='proto.MetricsData.userMetrics', index=11, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_METRICSDATA_USERMETRICSENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=867, + serialized_end=1331, +) + + +_HEALTHCHECKRESULT = _descriptor.Descriptor( + name='HealthCheckResult', + full_name='proto.HealthCheckResult', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='success', full_name='proto.HealthCheckResult.success', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1333, + serialized_end=1369, +) + + +_METRICS_INSTANCEMETRICS = _descriptor.Descriptor( + name='InstanceMetrics', + full_name='proto.Metrics.InstanceMetrics', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='proto.Metrics.InstanceMetrics.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='instanceId', full_name='proto.Metrics.InstanceMetrics.instanceId', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='metricsData', full_name='proto.Metrics.InstanceMetrics.metricsData', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1432, + serialized_end=1524, +) + +_METRICS = _descriptor.Descriptor( + name='Metrics', + full_name='proto.Metrics', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='metrics', full_name='proto.Metrics.metrics', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_METRICS_INSTANCEMETRICS, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1372, + serialized_end=1524, +) + +_FUNCTIONSTATUS_EXCEPTIONINFORMATION.containing_type = _FUNCTIONSTATUS +_FUNCTIONSTATUS.fields_by_name['latestUserExceptions'].message_type = _FUNCTIONSTATUS_EXCEPTIONINFORMATION +_FUNCTIONSTATUS.fields_by_name['latestSystemExceptions'].message_type = _FUNCTIONSTATUS_EXCEPTIONINFORMATION +_FUNCTIONSTATUS.fields_by_name['latestSourceExceptions'].message_type = _FUNCTIONSTATUS_EXCEPTIONINFORMATION +_FUNCTIONSTATUS.fields_by_name['latestSinkExceptions'].message_type = _FUNCTIONSTATUS_EXCEPTIONINFORMATION +_FUNCTIONSTATUSLIST.fields_by_name['functionStatusList'].message_type = _FUNCTIONSTATUS +_METRICSDATA_USERMETRICSENTRY.containing_type = _METRICSDATA +_METRICSDATA.fields_by_name['userMetrics'].message_type = _METRICSDATA_USERMETRICSENTRY +_METRICS_INSTANCEMETRICS.fields_by_name['metricsData'].message_type = _METRICSDATA +_METRICS_INSTANCEMETRICS.containing_type = _METRICS +_METRICS.fields_by_name['metrics'].message_type = _METRICS_INSTANCEMETRICS +DESCRIPTOR.message_types_by_name['FunctionStatus'] = _FUNCTIONSTATUS +DESCRIPTOR.message_types_by_name['FunctionStatusList'] = _FUNCTIONSTATUSLIST +DESCRIPTOR.message_types_by_name['MetricsData'] = _METRICSDATA +DESCRIPTOR.message_types_by_name['HealthCheckResult'] = _HEALTHCHECKRESULT +DESCRIPTOR.message_types_by_name['Metrics'] = _METRICS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +FunctionStatus = _reflection.GeneratedProtocolMessageType('FunctionStatus', (_message.Message,), { + + 'ExceptionInformation' : _reflection.GeneratedProtocolMessageType('ExceptionInformation', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTIONSTATUS_EXCEPTIONINFORMATION, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionStatus.ExceptionInformation) + }) + , + 'DESCRIPTOR' : _FUNCTIONSTATUS, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionStatus) + }) +_sym_db.RegisterMessage(FunctionStatus) +_sym_db.RegisterMessage(FunctionStatus.ExceptionInformation) + +FunctionStatusList = _reflection.GeneratedProtocolMessageType('FunctionStatusList', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTIONSTATUSLIST, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.FunctionStatusList) + }) +_sym_db.RegisterMessage(FunctionStatusList) + +MetricsData = _reflection.GeneratedProtocolMessageType('MetricsData', (_message.Message,), { + + 'UserMetricsEntry' : _reflection.GeneratedProtocolMessageType('UserMetricsEntry', (_message.Message,), { + 'DESCRIPTOR' : _METRICSDATA_USERMETRICSENTRY, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.MetricsData.UserMetricsEntry) + }) + , + 'DESCRIPTOR' : _METRICSDATA, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.MetricsData) + }) +_sym_db.RegisterMessage(MetricsData) +_sym_db.RegisterMessage(MetricsData.UserMetricsEntry) + +HealthCheckResult = _reflection.GeneratedProtocolMessageType('HealthCheckResult', (_message.Message,), { + 'DESCRIPTOR' : _HEALTHCHECKRESULT, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.HealthCheckResult) + }) +_sym_db.RegisterMessage(HealthCheckResult) + +Metrics = _reflection.GeneratedProtocolMessageType('Metrics', (_message.Message,), { + + 'InstanceMetrics' : _reflection.GeneratedProtocolMessageType('InstanceMetrics', (_message.Message,), { + 'DESCRIPTOR' : _METRICS_INSTANCEMETRICS, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.Metrics.InstanceMetrics) + }) + , + 'DESCRIPTOR' : _METRICS, + '__module__' : 'InstanceCommunication_pb2' + # @@protoc_insertion_point(class_scope:proto.Metrics) + }) +_sym_db.RegisterMessage(Metrics) +_sym_db.RegisterMessage(Metrics.InstanceMetrics) + + +DESCRIPTOR._options = None +_METRICSDATA_USERMETRICSENTRY._options = None + +_INSTANCECONTROL = _descriptor.ServiceDescriptor( + name='InstanceControl', + full_name='proto.InstanceControl', + file=DESCRIPTOR, + index=0, + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_start=1527, + serialized_end=1875, + methods=[ + _descriptor.MethodDescriptor( + name='GetFunctionStatus', + full_name='proto.InstanceControl.GetFunctionStatus', + index=0, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_FUNCTIONSTATUS, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='GetAndResetMetrics', + full_name='proto.InstanceControl.GetAndResetMetrics', + index=1, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_METRICSDATA, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='ResetMetrics', + full_name='proto.InstanceControl.ResetMetrics', + index=2, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='GetMetrics', + full_name='proto.InstanceControl.GetMetrics', + index=3, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_METRICSDATA, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='HealthCheck', + full_name='proto.InstanceControl.HealthCheck', + index=4, + containing_service=None, + input_type=google_dot_protobuf_dot_empty__pb2._EMPTY, + output_type=_HEALTHCHECKRESULT, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), +]) +_sym_db.RegisterServiceDescriptor(_INSTANCECONTROL) + +DESCRIPTOR.services_by_name['InstanceControl'] = _INSTANCECONTROL + +# @@protoc_insertion_point(module_scope) diff --git a/conf/instances/python-instance/InstanceCommunication_pb2_grpc.py b/conf/instances/python-instance/InstanceCommunication_pb2_grpc.py new file mode 100644 index 0000000000000..e895f53dbfa9f --- /dev/null +++ b/conf/instances/python-instance/InstanceCommunication_pb2_grpc.py @@ -0,0 +1,218 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import InstanceCommunication_pb2 as InstanceCommunication__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + + +class InstanceControlStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetFunctionStatus = channel.unary_unary( + '/proto.InstanceControl/GetFunctionStatus', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=InstanceCommunication__pb2.FunctionStatus.FromString, + ) + self.GetAndResetMetrics = channel.unary_unary( + '/proto.InstanceControl/GetAndResetMetrics', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=InstanceCommunication__pb2.MetricsData.FromString, + ) + self.ResetMetrics = channel.unary_unary( + '/proto.InstanceControl/ResetMetrics', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + ) + self.GetMetrics = channel.unary_unary( + '/proto.InstanceControl/GetMetrics', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=InstanceCommunication__pb2.MetricsData.FromString, + ) + self.HealthCheck = channel.unary_unary( + '/proto.InstanceControl/HealthCheck', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=InstanceCommunication__pb2.HealthCheckResult.FromString, + ) + + +class InstanceControlServicer(object): + """Missing associated documentation comment in .proto file.""" + + def GetFunctionStatus(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetAndResetMetrics(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ResetMetrics(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetMetrics(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HealthCheck(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_InstanceControlServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetFunctionStatus': grpc.unary_unary_rpc_method_handler( + servicer.GetFunctionStatus, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=InstanceCommunication__pb2.FunctionStatus.SerializeToString, + ), + 'GetAndResetMetrics': grpc.unary_unary_rpc_method_handler( + servicer.GetAndResetMetrics, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=InstanceCommunication__pb2.MetricsData.SerializeToString, + ), + 'ResetMetrics': grpc.unary_unary_rpc_method_handler( + servicer.ResetMetrics, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'GetMetrics': grpc.unary_unary_rpc_method_handler( + servicer.GetMetrics, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=InstanceCommunication__pb2.MetricsData.SerializeToString, + ), + 'HealthCheck': grpc.unary_unary_rpc_method_handler( + servicer.HealthCheck, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=InstanceCommunication__pb2.HealthCheckResult.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'proto.InstanceControl', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class InstanceControl(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetFunctionStatus(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.InstanceControl/GetFunctionStatus', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + InstanceCommunication__pb2.FunctionStatus.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetAndResetMetrics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.InstanceControl/GetAndResetMetrics', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + InstanceCommunication__pb2.MetricsData.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ResetMetrics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.InstanceControl/ResetMetrics', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetMetrics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.InstanceControl/GetMetrics', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + InstanceCommunication__pb2.MetricsData.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HealthCheck(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.InstanceControl/HealthCheck', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + InstanceCommunication__pb2.HealthCheckResult.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/conf/instances/python-instance/__init__.py b/conf/instances/python-instance/__init__.py new file mode 100644 index 0000000000000..d70b15fbd168e --- /dev/null +++ b/conf/instances/python-instance/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- \ No newline at end of file diff --git a/conf/instances/python-instance/contextimpl.py b/conf/instances/python-instance/contextimpl.py new file mode 100644 index 0000000000000..27618626cf77e --- /dev/null +++ b/conf/instances/python-instance/contextimpl.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""contextimpl.py: ContextImpl class that implements the Context interface +""" + +import re +import time +import os +import json +import log + +import pulsar +import util + +from prometheus_client import Summary +from function_stats import Stats +from functools import partial + +Log = log.Log + +class ContextImpl(pulsar.Context): + + # add label to indicate user metric + user_metrics_label_names = Stats.metrics_label_names + ["metric"] + + def __init__(self, instance_config, logger, pulsar_client, user_code, consumers, + secrets_provider, metrics_labels, state_context, stats): + self.instance_config = instance_config + self.log = logger + self.pulsar_client = pulsar_client + self.user_code_dir = os.path.dirname(user_code) + self.consumers = consumers + self.secrets_provider = secrets_provider + self.state_context = state_context + self.publish_producers = {} + self.publish_serializers = {} + self.message = None + self.current_start_time = None + self.user_config = json.loads(instance_config.function_details.userConfig) \ + if instance_config.function_details.userConfig \ + else [] + self.secrets_map = json.loads(instance_config.function_details.secretsMap) \ + if instance_config.function_details.secretsMap \ + else {} + + self.metrics_labels = metrics_labels + self.user_metrics_map = dict() + self.user_metrics_summary = Summary("pulsar_function_user_metric", + 'Pulsar Function user defined metric', + ContextImpl.user_metrics_label_names) + self.stats = stats + + # Called on a per message basis to set the context for the current message + def set_current_message_context(self, message, topic): + self.message = message + self.current_start_time = time.time() + + def get_message_id(self): + return self.message.message_id() + + def get_message_key(self): + return self.message.partition_key() + + def get_message_eventtime(self): + return self.message.event_timestamp() + + def get_message_properties(self): + return self.message.properties() + + def get_current_message_topic_name(self): + return self.message.topic_name() + + def get_partition_key(self): + return self.message.partition_key() + + def get_function_name(self): + return self.instance_config.function_details.name + + def get_function_tenant(self): + return self.instance_config.function_details.tenant + + def get_function_namespace(self): + return self.instance_config.function_details.namespace + + def get_function_id(self): + return self.instance_config.function_id + + def get_instance_id(self): + return self.instance_config.instance_id + + def get_function_version(self): + return self.instance_config.function_version + + def get_logger(self): + return self.log + + def get_user_config_value(self, key): + if key in self.user_config: + return self.user_config[key] + else: + return None + + def get_user_config_map(self): + return self.user_config + + def get_secret(self, secret_key): + if not secret_key in self.secrets_map: + return None + return self.secrets_provider.provide_secret(secret_key, self.secrets_map[secret_key]) + + def record_metric(self, metric_name, metric_value): + if metric_name not in self.user_metrics_map: + user_metrics_labels = self.metrics_labels + [metric_name] + self.user_metrics_map[metric_name] = self.user_metrics_summary.labels(*user_metrics_labels) + + self.user_metrics_map[metric_name].observe(metric_value) + + def get_input_topics(self): + return list(self.instance_config.function_details.source.inputSpecs.keys()) + + def get_output_topic(self): + return self.instance_config.function_details.sink.topic + + def get_output_serde_class_name(self): + return self.instance_config.function_details.sink.serDeClassName + + def callback_wrapper(self, callback, topic, message_id, result, msg): + if result != pulsar.Result.Ok: + error_msg = "Failed to publish to topic [%s] with error [%s] with src message id [%s]" % (topic, result, message_id) + Log.error(error_msg) + self.stats.incr_total_sys_exceptions(Exception(error_msg)) + if callback: + callback(result, msg) + + def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None, message_conf=None): + # Just make sure that user supplied values are properly typed + topic_name = str(topic_name) + serde_class_name = str(serde_class_name) + pulsar_compression_type = pulsar._pulsar.CompressionType.NONE + if compression_type is not None: + pulsar_compression_type = compression_type + if topic_name not in self.publish_producers: + self.publish_producers[topic_name] = self.pulsar_client.create_producer( + topic_name, + block_if_queue_full=True, + batching_enabled=True, + batching_max_publish_delay_ms=10, + compression_type=pulsar_compression_type, + properties=util.get_properties(util.getFullyQualifiedFunctionName( + self.instance_config.function_details.tenant, + self.instance_config.function_details.namespace, + self.instance_config.function_details.name), + self.instance_config.instance_id) + ) + + if serde_class_name not in self.publish_serializers: + serde_klass = util.import_class(self.user_code_dir, serde_class_name) + self.publish_serializers[serde_class_name] = serde_klass() + + output_bytes = bytes(self.publish_serializers[serde_class_name].serialize(message)) + + if properties: + # The deprecated properties args was passed. Need to merge into message_conf + if not message_conf: + message_conf = {} + message_conf['properties'] = properties + + if message_conf: + self.publish_producers[topic_name].send_async( + output_bytes, partial(self.callback_wrapper, callback, topic_name, self.get_message_id()), **message_conf) + else: + self.publish_producers[topic_name].send_async( + output_bytes, partial(self.callback_wrapper, callback, topic_name, self.get_message_id())) + + def ack(self, msgid, topic): + topic_consumer = None + if topic in self.consumers: + topic_consumer = self.consumers[topic] + else: + # if this topic is a partitioned topic + m = re.search('(.+)-partition-(\d+)', topic) + if not m: + raise ValueError('Invalid topicname %s' % topic) + elif m.group(1) in self.consumers: + topic_consumer = self.consumers[m.group(1)] + else: + raise ValueError('Invalid topicname %s' % topic) + topic_consumer.acknowledge(msgid) + + def get_and_reset_metrics(self): + metrics = self.get_metrics() + # TODO(sanjeev):- Make this thread safe + self.reset_metrics() + return metrics + + def reset_metrics(self): + # TODO: Make it thread safe + for user_metric in self.user_metrics_map.values(): + user_metric._sum.set(0.0) + user_metric._count.set(0.0) + + def get_metrics(self): + metrics_map = {} + for metric_name, user_metric in self.user_metrics_map.items(): + metrics_map["%s%s_sum" % (Stats.USER_METRIC_PREFIX, metric_name)] = user_metric._sum.get() + metrics_map["%s%s_count" % (Stats.USER_METRIC_PREFIX, metric_name)] = user_metric._count.get() + + return metrics_map + + def incr_counter(self, key, amount): + return self.state_context.incr(key, amount) + + def get_counter(self, key): + return self.state_context.get_amount(key) + + def del_counter(self, key): + return self.state_context.delete(key) + + def put_state(self, key, value): + return self.state_context.put(key, value) + + def get_state(self, key): + return self.state_context.get_value(key) diff --git a/conf/instances/python-instance/function_stats.py b/conf/instances/python-instance/function_stats.py new file mode 100644 index 0000000000000..8b54f75044de2 --- /dev/null +++ b/conf/instances/python-instance/function_stats.py @@ -0,0 +1,221 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import traceback +import time +import util +import sys + +from prometheus_client import Counter, Summary, Gauge +from ratelimit import limits, RateLimitException + +# We keep track of the following metrics +class Stats(object): + metrics_label_names = ['tenant', 'namespace', 'name', 'instance_id', 'cluster', 'fqfn'] + + exception_metrics_label_names = metrics_label_names + ['error'] + + PULSAR_FUNCTION_METRICS_PREFIX = "pulsar_function_" + USER_METRIC_PREFIX = "user_metric_" + + TOTAL_SUCCESSFULLY_PROCESSED = 'processed_successfully_total' + TOTAL_SYSTEM_EXCEPTIONS = 'system_exceptions_total' + TOTAL_USER_EXCEPTIONS = 'user_exceptions_total' + PROCESS_LATENCY_MS = 'process_latency_ms' + LAST_INVOCATION = 'last_invocation' + TOTAL_RECEIVED = 'received_total' + + TOTAL_SUCCESSFULLY_PROCESSED_1min = 'processed_successfully_total_1min' + TOTAL_SYSTEM_EXCEPTIONS_1min = 'system_exceptions_total_1min' + TOTAL_USER_EXCEPTIONS_1min = 'user_exceptions_total_1min' + PROCESS_LATENCY_MS_1min = 'process_latency_ms_1min' + TOTAL_RECEIVED_1min = 'received_total_1min' + + # Declare Prometheus + stat_total_processed_successfully = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_SUCCESSFULLY_PROCESSED, + 'Total number of messages processed successfully.', metrics_label_names) + stat_total_sys_exceptions = Counter(PULSAR_FUNCTION_METRICS_PREFIX+ TOTAL_SYSTEM_EXCEPTIONS, 'Total number of system exceptions.', + metrics_label_names) + stat_total_user_exceptions = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_USER_EXCEPTIONS, 'Total number of user exceptions.', + metrics_label_names) + + stat_process_latency_ms = Summary(PULSAR_FUNCTION_METRICS_PREFIX + PROCESS_LATENCY_MS, 'Process latency in milliseconds.', metrics_label_names) + + stat_last_invocation = Gauge(PULSAR_FUNCTION_METRICS_PREFIX + LAST_INVOCATION, 'The timestamp of the last invocation of the function.', metrics_label_names) + + stat_total_received = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_RECEIVED, 'Total number of messages received from source.', metrics_label_names) + + # 1min windowed metrics + stat_total_processed_successfully_1min = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_SUCCESSFULLY_PROCESSED_1min, + 'Total number of messages processed successfully in the last 1 minute.', metrics_label_names) + stat_total_sys_exceptions_1min = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_SYSTEM_EXCEPTIONS_1min, + 'Total number of system exceptions in the last 1 minute.', + metrics_label_names) + stat_total_user_exceptions_1min = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_USER_EXCEPTIONS_1min, + 'Total number of user exceptions in the last 1 minute.', + metrics_label_names) + + stat_process_latency_ms_1min = Summary(PULSAR_FUNCTION_METRICS_PREFIX + PROCESS_LATENCY_MS_1min, + 'Process latency in milliseconds in the last 1 minute.', metrics_label_names) + + stat_total_received_1min = Counter(PULSAR_FUNCTION_METRICS_PREFIX + TOTAL_RECEIVED_1min, + 'Total number of messages received from source in the last 1 minute.', metrics_label_names) + + # exceptions + user_exceptions = Gauge(PULSAR_FUNCTION_METRICS_PREFIX + 'user_exception', 'Exception from user code.', exception_metrics_label_names) + + system_exceptions = Gauge(PULSAR_FUNCTION_METRICS_PREFIX + 'system_exception', 'Exception from system code.', exception_metrics_label_names) + + latest_user_exception = [] + latest_sys_exception = [] + + def __init__(self, metrics_labels): + self.metrics_labels = metrics_labels + self.process_start_time = None + + # as optimization + self._stat_total_processed_successfully = self.stat_total_processed_successfully.labels(*self.metrics_labels) + self._stat_total_sys_exceptions = self.stat_total_sys_exceptions.labels(*self.metrics_labels) + self._stat_total_user_exceptions = self.stat_total_user_exceptions.labels(*self.metrics_labels) + self._stat_process_latency_ms = self.stat_process_latency_ms.labels(*self.metrics_labels) + self._stat_last_invocation = self.stat_last_invocation.labels(*self.metrics_labels) + self._stat_total_received = self.stat_total_received.labels(*self.metrics_labels) + self._stat_total_processed_successfully_1min = self.stat_total_processed_successfully_1min.labels(*self.metrics_labels) + self._stat_total_sys_exceptions_1min = self.stat_total_sys_exceptions_1min.labels(*self.metrics_labels) + self._stat_total_user_exceptions_1min = self.stat_total_user_exceptions_1min.labels(*self.metrics_labels) + self._stat_process_latency_ms_1min = self.stat_process_latency_ms_1min.labels(*self.metrics_labels) + self._stat_total_received_1min = self.stat_total_received_1min.labels(*self.metrics_labels) + + # start time for windowed metrics + util.FixedTimer(60, self.reset, name="windowed-metrics-timer").start() + + def get_total_received(self): + return self._stat_total_received._value.get() + + def get_total_processed_successfully(self): + return self._stat_total_processed_successfully._value.get() + + def get_total_sys_exceptions(self): + return self._stat_total_sys_exceptions._value.get() + + def get_total_user_exceptions(self): + return self._stat_total_user_exceptions._value.get() + + def get_avg_process_latency(self): + process_latency_ms_count = self._stat_process_latency_ms._count.get() + process_latency_ms_sum = self._stat_process_latency_ms._sum.get() + return 0.0 \ + if process_latency_ms_count <= 0.0 \ + else process_latency_ms_sum / process_latency_ms_count + + def get_total_processed_successfully_1min(self): + return self._stat_total_processed_successfully_1min._value.get() + + def get_total_sys_exceptions_1min(self): + return self._stat_total_sys_exceptions_1min._value.get() + + def get_total_user_exceptions_1min(self): + return self._stat_total_user_exceptions_1min._value.get() + + def get_total_received_1min(self): + return self._stat_total_received_1min._value.get() + + def get_avg_process_latency_1min(self): + process_latency_ms_count = self._stat_process_latency_ms_1min._count.get() + process_latency_ms_sum = self._stat_process_latency_ms_1min._sum.get() + return 0.0 \ + if process_latency_ms_count <= 0.0 \ + else process_latency_ms_sum / process_latency_ms_count + + def get_last_invocation(self): + return self._stat_last_invocation._value.get() + + def incr_total_processed_successfully(self): + self._stat_total_processed_successfully.inc() + self._stat_total_processed_successfully_1min.inc() + + def incr_total_sys_exceptions(self, exception): + self._stat_total_sys_exceptions.inc() + self._stat_total_sys_exceptions_1min.inc() + self.add_sys_exception(exception) + + def incr_total_user_exceptions(self, exception): + self._stat_total_user_exceptions.inc() + self._stat_total_user_exceptions_1min.inc() + self.add_user_exception(exception) + + def incr_total_received(self): + self._stat_total_received.inc() + self._stat_total_received_1min.inc() + + def process_time_start(self): + self.process_start_time = time.time() + + def process_time_end(self): + if self.process_start_time: + duration = (time.time() - self.process_start_time) * 1000.0 + self._stat_process_latency_ms.observe(duration) + self._stat_process_latency_ms_1min.observe(duration) + + def set_last_invocation(self, time): + self._stat_last_invocation.set(time * 1000.0) + + def add_user_exception(self, exception): + error = traceback.format_exc() + ts = int(time.time() * 1000) if sys.version_info.major >= 3 else long(time.time() * 1000) + self.latest_user_exception.append((error, ts)) + if len(self.latest_user_exception) > 10: + self.latest_user_exception.pop(0) + + # report exception via prometheus + try: + self.report_user_exception_prometheus(exception) + except RateLimitException: + pass + + @limits(calls=5, period=60) + def report_user_exception_prometheus(self, exception): + exception_metric_labels = self.metrics_labels + [str(exception)] + self.user_exceptions.labels(*exception_metric_labels).set(1.0) + + def add_sys_exception(self, exception): + error = traceback.format_exc() + ts = int(time.time() * 1000) if sys.version_info.major >= 3 else long(time.time() * 1000) + self.latest_sys_exception.append((error, ts)) + if len(self.latest_sys_exception) > 10: + self.latest_sys_exception.pop(0) + + # report exception via prometheus + try: + self.report_system_exception_prometheus(exception) + except RateLimitException: + pass + + @limits(calls=5, period=60) + def report_system_exception_prometheus(self, exception): + exception_metric_labels = self.metrics_labels + [str(exception)] + self.system_exceptions.labels(*exception_metric_labels).set(1.0) + + def reset(self): + self._stat_total_processed_successfully_1min._value.set(0.0) + self._stat_total_user_exceptions_1min._value.set(0.0) + self._stat_total_sys_exceptions_1min._value.set(0.0) + self._stat_process_latency_ms_1min._sum.set(0.0) + self._stat_process_latency_ms_1min._count.set(0.0) + self._stat_total_received_1min._value.set(0.0) diff --git a/conf/instances/python-instance/log.py b/conf/instances/python-instance/log.py new file mode 100644 index 0000000000000..5dfee700bbae5 --- /dev/null +++ b/conf/instances/python-instance/log.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +''' log.py ''' +import logging +import logging.config +import logging.handlers +import os +import errno +import pulsar +import sys + +# Create the logger +# pylint: disable=invalid-name +logging.basicConfig() +Log = logging.getLogger() + +# time formatter - date - time - UTC offset +# e.g. "08/16/1988 21:30:00 +1030" +# see time formatter documentation for more +date_format = "%Y-%m-%d %H:%M:%S %z" + +class LogTopicHandler(logging.Handler): + def __init__(self, topic_name, pulsar_client): + logging.Handler.__init__(self) + Log.info("Setting up producer for log topic %s" % topic_name) + self.producer = pulsar_client.create_producer( + str(topic_name), + block_if_queue_full=True, + batching_enabled=True, + batching_max_publish_delay_ms=100, + compression_type=pulsar._pulsar.CompressionType.LZ4) + + def emit(self, record): + msg = self.format(record) + self.producer.send_async(str(msg).encode('utf-8'), None) + +def mkdir_p(path): + try: + os.makedirs(path, exist_ok=True) # Python>3.2 + except TypeError: + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +# logging handler that is RotatingFileHandler but creates path to log file for you +# if it doesn't exist +class CreatePathRotatingFileHandler(logging.handlers.RotatingFileHandler): + def __init__(self, filename, mode='a', maxBytes=10 * 1024 * 1024, backupCount=5, encoding=None, delay=0): + mkdir_p(os.path.dirname(filename)) + logging.handlers.RotatingFileHandler.__init__(self, filename, mode=mode, maxBytes=maxBytes, backupCount=backupCount, encoding=encoding, delay=delay) + +def remove_all_handlers(): + retval = None + for handler in Log.handlers: + Log.handlers.remove(handler) + retval = handler + return retval + +def add_handler(stream_handler): + log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" + formatter = logging.Formatter(fmt=log_format, datefmt=date_format) + stream_handler.setFormatter(formatter) + Log.addHandler(stream_handler) + +def init_logger(level, logfile, logging_config_file): + global Log + # get log file location for function instance + os.environ['LOG_FILE'] = logfile + logging.config.fileConfig(logging_config_file) + Log = logging.getLogger() + Log.setLevel(level) + + # set print to redirect to logger + class StreamToLogger(object): + """ + Fake file-like stream object that redirects writes to a logger instance. + """ + + def __init__(self, logger, log_level=logging.INFO): + self.logger = logger + self.log_level = log_level + self.linebuf = '' + + def write(self, buf): + for line in buf.rstrip().splitlines(): + self.logger.log(self.log_level, line.rstrip()) + + def flush(self): + pass + + sl = StreamToLogger(Log, logging.INFO) + sys.stdout = sl + + sl = StreamToLogger(Log, logging.ERROR) + sys.stderr = sl diff --git a/conf/instances/python-instance/prometheus_client_fix.py b/conf/instances/python-instance/prometheus_client_fix.py new file mode 100644 index 0000000000000..17d9ffedf0990 --- /dev/null +++ b/conf/instances/python-instance/prometheus_client_fix.py @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from __future__ import unicode_literals + +import threading + +from prometheus_client import core +from prometheus_client.exposition import MetricsHandler + +try: + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + from SocketServer import ThreadingMixIn + from urllib2 import build_opener, Request, HTTPHandler + from urllib import quote_plus + from urlparse import parse_qs, urlparse +except ImportError: + # Python 3 + from http.server import BaseHTTPRequestHandler, HTTPServer + from socketserver import ThreadingMixIn + from urllib.request import build_opener, Request, HTTPHandler + from urllib.parse import quote_plus, parse_qs, urlparse + + +class _ThreadingSimpleServer(ThreadingMixIn, HTTPServer): + """Thread per request HTTP server.""" + # Make worker threads "fire and forget". Beginning with Python 3.7 this + # prevents a memory leak because ``ThreadingMixIn`` starts to gather all + # non-daemon threads in a list in order to join on them at server close. + # Enabling daemon threads virtually makes ``_ThreadingSimpleServer`` the + # same as Python 3.7's ``ThreadingHTTPServer``. + daemon_threads = True + +def start_http_server(port, addr='', registry=core.REGISTRY): + """Starts an HTTP server for prometheus metrics as a daemon thread""" + CustomMetricsHandler = MetricsHandler.factory(registry) + httpd = _ThreadingSimpleServer((addr, port), CustomMetricsHandler) + t = threading.Thread(target=httpd.serve_forever) + t.daemon = True + t.start() \ No newline at end of file diff --git a/conf/instances/python-instance/pulsar/__init__.py b/conf/instances/python-instance/pulsar/__init__.py new file mode 100644 index 0000000000000..3773702937088 --- /dev/null +++ b/conf/instances/python-instance/pulsar/__init__.py @@ -0,0 +1,1356 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +""" +The Pulsar Python client library is based on the existing C++ client library. +All the same features are exposed through the Python interface. + +Currently, the supported Python versions are 2.7, 3.5, 3.6, 3.7 and 3.8. + +## Install from PyPI + +Download Python wheel binary files for MacOS and Linux +directly from the PyPI archive. + + #!shell + $ sudo pip install pulsar-client + +## Install from sources + +Follow the instructions to compile the Pulsar C++ client library. This method +will also build the Python binding for the library. + +To install the Python bindings: + + #!shell + $ cd pulsar-client-cpp/python + $ sudo python setup.py install + +## Examples + +### [Producer](#pulsar.Producer) example + + #!python + import pulsar + + client = pulsar.Client('pulsar://localhost:6650') + + producer = client.create_producer('my-topic') + + for i in range(10): + producer.send(('Hello-%d' % i).encode('utf-8')) + + client.close() + +#### [Consumer](#pulsar.Consumer) Example + + #!python + import pulsar + + client = pulsar.Client('pulsar://localhost:6650') + consumer = client.subscribe('my-topic', 'my-subscription') + + while True: + msg = consumer.receive() + try: + print("Received message '%s' id='%s'", msg.data().decode('utf-8'), msg.message_id()) + consumer.acknowledge(msg) + except: + consumer.negative_acknowledge(msg) + + client.close() + +### [Async producer](#pulsar.Producer.send_async) example + + #!python + import pulsar + + client = pulsar.Client('pulsar://localhost:6650') + + producer = client.create_producer( + 'my-topic', + block_if_queue_full=True, + batching_enabled=True, + batching_max_publish_delay_ms=10 + ) + + def send_callback(res, msg_id): + print('Message published res=%s', res) + + while True: + producer.send_async(('Hello-%d' % i).encode('utf-8'), send_callback) + + client.close() +""" + +import logging +import _pulsar + +from _pulsar import Result, CompressionType, ConsumerType, InitialPosition, PartitionsRoutingMode, BatchingType # noqa: F401 + +from pulsar.exceptions import * + +from pulsar.functions.function import Function +from pulsar.functions.context import Context +from pulsar.functions.serde import SerDe, IdentitySerDe, PickleSerDe +from pulsar import schema +_schema = schema + +import re +_retype = type(re.compile('x')) + +import certifi +from datetime import timedelta + + +class MessageId: + """ + Represents a message id + """ + + def __init__(self, partition=-1, ledger_id=-1, entry_id=-1, batch_index=-1): + self._msg_id = _pulsar.MessageId(partition, ledger_id, entry_id, batch_index) + + 'Represents the earliest message stored in a topic' + earliest = _pulsar.MessageId.earliest + + 'Represents the latest message published on a topic' + latest = _pulsar.MessageId.latest + + def ledger_id(self): + return self._msg_id.ledger_id() + + def entry_id(self): + return self._msg_id.entry_id() + + def batch_index(self): + return self._msg_id.batch_index() + + def partition(self): + return self._msg_id.partition() + + def serialize(self): + """ + Returns a bytes representation of the message id. + This bytes sequence can be stored and later deserialized. + """ + return self._msg_id.serialize() + + @staticmethod + def deserialize(message_id_bytes): + """ + Deserialize a message id object from a previously + serialized bytes sequence. + """ + return _pulsar.MessageId.deserialize(message_id_bytes) + + +class Message: + """ + Message objects are returned by a consumer, either by calling `receive` or + through a listener. + """ + + def data(self): + """ + Returns object typed bytes with the payload of the message. + """ + return self._message.data() + + def value(self): + """ + Returns object with the de-serialized version of the message content + """ + return self._schema.decode(self._message.data()) + + def properties(self): + """ + Return the properties attached to the message. Properties are + application-defined key/value pairs that will be attached to the + message. + """ + return self._message.properties() + + def partition_key(self): + """ + Get the partitioning key for the message. + """ + return self._message.partition_key() + + def publish_timestamp(self): + """ + Get the timestamp in milliseconds with the message publish time. + """ + return self._message.publish_timestamp() + + def event_timestamp(self): + """ + Get the timestamp in milliseconds with the message event time. + """ + return self._message.event_timestamp() + + def message_id(self): + """ + The message ID that can be used to refere to this particular message. + """ + return self._message.message_id() + + def topic_name(self): + """ + Get the topic Name from which this message originated from + """ + return self._message.topic_name() + + def redelivery_count(self): + """ + Get the redelivery count for this message + """ + return self._message.redelivery_count() + + def schema_version(self): + """ + Get the schema version for this message + """ + return self._message.schema_version() + + @staticmethod + def _wrap(_message): + self = Message() + self._message = _message + return self + + +class MessageBatch: + + def __init__(self): + self._msg_batch = _pulsar.MessageBatch() + + def with_message_id(self, msg_id): + if not isinstance(msg_id, _pulsar.MessageId): + if isinstance(msg_id, MessageId): + msg_id = msg_id._msg_id + else: + raise TypeError("unknown message id type") + self._msg_batch.with_message_id(msg_id) + return self + + def parse_from(self, data, size): + self._msg_batch.parse_from(data, size) + _msgs = self._msg_batch.messages() + return list(map(Message._wrap, _msgs)) + + +class Authentication: + """ + Authentication provider object. Used to load authentication from an external + shared library. + """ + def __init__(self, dynamicLibPath, authParamsString): + """ + Create the authentication provider instance. + + **Args** + + * `dynamicLibPath`: Path to the authentication provider shared library + (such as `tls.so`) + * `authParamsString`: Comma-separated list of provider-specific + configuration params + """ + _check_type(str, dynamicLibPath, 'dynamicLibPath') + _check_type(str, authParamsString, 'authParamsString') + self.auth = _pulsar.Authentication(dynamicLibPath, authParamsString) + + +class AuthenticationTLS(Authentication): + """ + TLS Authentication implementation + """ + def __init__(self, certificate_path, private_key_path): + """ + Create the TLS authentication provider instance. + + **Args** + + * `certificatePath`: Path to the public certificate + * `privateKeyPath`: Path to private TLS key + """ + _check_type(str, certificate_path, 'certificate_path') + _check_type(str, private_key_path, 'private_key_path') + self.auth = _pulsar.AuthenticationTLS(certificate_path, private_key_path) + + +class AuthenticationToken(Authentication): + """ + Token based authentication implementation + """ + def __init__(self, token): + """ + Create the token authentication provider instance. + + **Args** + + * `token`: A string containing the token or a functions that provides a + string with the token + """ + if not (isinstance(token, str) or callable(token)): + raise ValueError("Argument token is expected to be of type 'str' or a function returning 'str'") + self.auth = _pulsar.AuthenticationToken(token) + + +class AuthenticationAthenz(Authentication): + """ + Athenz Authentication implementation + """ + def __init__(self, auth_params_string): + """ + Create the Athenz authentication provider instance. + + **Args** + + * `auth_params_string`: JSON encoded configuration for Athenz client + """ + _check_type(str, auth_params_string, 'auth_params_string') + self.auth = _pulsar.AuthenticationAthenz(auth_params_string) + +class AuthenticationOauth2(Authentication): + """ + Oauth2 Authentication implementation + """ + def __init__(self, auth_params_string): + """ + Create the Oauth2 authentication provider instance. + + **Args** + + * `auth_params_string`: JSON encoded configuration for Oauth2 client + """ + _check_type(str, auth_params_string, 'auth_params_string') + self.auth = _pulsar.AuthenticationOauth2(auth_params_string) + +class Client: + """ + The Pulsar client. A single client instance can be used to create producers + and consumers on multiple topics. + + The client will share the same connection pool and threads across all + producers and consumers. + """ + + def __init__(self, service_url, + authentication=None, + operation_timeout_seconds=30, + io_threads=1, + message_listener_threads=1, + concurrent_lookup_requests=50000, + log_conf_file_path=None, + use_tls=False, + tls_trust_certs_file_path=None, + tls_allow_insecure_connection=False, + tls_validate_hostname=False, + logger=None, + connection_timeout_ms=10000, + ): + """ + Create a new Pulsar client instance. + + **Args** + + * `service_url`: The Pulsar service url eg: pulsar://my-broker.com:6650/ + + **Options** + + * `authentication`: + Set the authentication provider to be used with the broker. For example: + `AuthenticationTls`, AuthenticaionToken, `AuthenticationAthenz`or `AuthenticationOauth2` + * `operation_timeout_seconds`: + Set timeout on client operations (subscribe, create producer, close, + unsubscribe). + * `io_threads`: + Set the number of IO threads to be used by the Pulsar client. + * `message_listener_threads`: + Set the number of threads to be used by the Pulsar client when + delivering messages through message listener. The default is 1 thread + per Pulsar client. If using more than 1 thread, messages for distinct + `message_listener`s will be delivered in different threads, however a + single `MessageListener` will always be assigned to the same thread. + * `concurrent_lookup_requests`: + Number of concurrent lookup-requests allowed on each broker connection + to prevent overload on the broker. + * `log_conf_file_path`: + Initialize log4cxx from a configuration file. + * `use_tls`: + Configure whether to use TLS encryption on the connection. This setting + is deprecated. TLS will be automatically enabled if the `serviceUrl` is + set to `pulsar+ssl://` or `https://` + * `tls_trust_certs_file_path`: + Set the path to the trusted TLS certificate file. If empty defaults to + certifi. + * `tls_allow_insecure_connection`: + Configure whether the Pulsar client accepts untrusted TLS certificates + from the broker. + * `tls_validate_hostname`: + Configure whether the Pulsar client validates that the hostname of the + endpoint, matches the common name on the TLS certificate presented by + the endpoint. + * `logger`: + Set a Python logger for this Pulsar client. Should be an instance of `logging.Logger`. + * `connection_timeout_ms`: + Set timeout in milliseconds on TCP connections. + """ + _check_type(str, service_url, 'service_url') + _check_type_or_none(Authentication, authentication, 'authentication') + _check_type(int, operation_timeout_seconds, 'operation_timeout_seconds') + _check_type(int, connection_timeout_ms, 'connection_timeout_ms') + _check_type(int, io_threads, 'io_threads') + _check_type(int, message_listener_threads, 'message_listener_threads') + _check_type(int, concurrent_lookup_requests, 'concurrent_lookup_requests') + _check_type_or_none(str, log_conf_file_path, 'log_conf_file_path') + _check_type(bool, use_tls, 'use_tls') + _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') + _check_type(bool, tls_allow_insecure_connection, 'tls_allow_insecure_connection') + _check_type(bool, tls_validate_hostname, 'tls_validate_hostname') + _check_type_or_none(logging.Logger, logger, 'logger') + + conf = _pulsar.ClientConfiguration() + if authentication: + conf.authentication(authentication.auth) + conf.operation_timeout_seconds(operation_timeout_seconds) + conf.connection_timeout(connection_timeout_ms) + conf.io_threads(io_threads) + conf.message_listener_threads(message_listener_threads) + conf.concurrent_lookup_requests(concurrent_lookup_requests) + if log_conf_file_path: + conf.log_conf_file_path(log_conf_file_path) + if logger: + conf.set_logger(logger) + if use_tls or service_url.startswith('pulsar+ssl://') or service_url.startswith('https://'): + conf.use_tls(True) + if tls_trust_certs_file_path: + conf.tls_trust_certs_file_path(tls_trust_certs_file_path) + else: + conf.tls_trust_certs_file_path(certifi.where()) + conf.tls_allow_insecure_connection(tls_allow_insecure_connection) + conf.tls_validate_hostname(tls_validate_hostname) + self._client = _pulsar.Client(service_url, conf) + self._consumers = [] + + def create_producer(self, topic, + producer_name=None, + schema=schema.BytesSchema(), + initial_sequence_id=None, + send_timeout_millis=30000, + compression_type=CompressionType.NONE, + max_pending_messages=1000, + max_pending_messages_across_partitions=50000, + block_if_queue_full=False, + batching_enabled=False, + batching_max_messages=1000, + batching_max_allowed_size_in_bytes=128*1024, + batching_max_publish_delay_ms=10, + message_routing_mode=PartitionsRoutingMode.RoundRobinDistribution, + lazy_start_partitioned_producers=False, + properties=None, + batching_type=BatchingType.Default, + encryption_key=None, + crypto_key_reader=None + ): + """ + Create a new producer on a given topic. + + **Args** + + * `topic`: + The topic name + + **Options** + + * `producer_name`: + Specify a name for the producer. If not assigned, + the system will generate a globally unique name which can be accessed + with `Producer.producer_name()`. When specifying a name, it is app to + the user to ensure that, for a given topic, the producer name is unique + across all Pulsar's clusters. + * `schema`: + Define the schema of the data that will be published by this producer. + The schema will be used for two purposes: + - Validate the data format against the topic defined schema + - Perform serialization/deserialization between data and objects + An example for this parameter would be to pass `schema=JsonSchema(MyRecordClass)`. + * `initial_sequence_id`: + Set the baseline for the sequence ids for messages + published by the producer. First message will be using + `(initialSequenceId + 1)`` as its sequence id and subsequent messages will + be assigned incremental sequence ids, if not otherwise specified. + * `send_timeout_millis`: + If a message is not acknowledged by the server before the + `send_timeout` expires, an error will be reported. + * `compression_type`: + Set the compression type for the producer. By default, message + payloads are not compressed. Supported compression types are + `CompressionType.LZ4`, `CompressionType.ZLib`, `CompressionType.ZSTD` and `CompressionType.SNAPPY`. + ZSTD is supported since Pulsar 2.3. Consumers will need to be at least at that + release in order to be able to receive messages compressed with ZSTD. + SNAPPY is supported since Pulsar 2.4. Consumers will need to be at least at that + release in order to be able to receive messages compressed with SNAPPY. + * `max_pending_messages`: + Set the max size of the queue holding the messages pending to receive + an acknowledgment from the broker. + * `max_pending_messages_across_partitions`: + Set the max size of the queue holding the messages pending to receive + an acknowledgment across partitions from the broker. + * `block_if_queue_full`: Set whether `send_async` operations should + block when the outgoing message queue is full. + * `message_routing_mode`: + Set the message routing mode for the partitioned producer. Default is `PartitionsRoutingMode.RoundRobinDistribution`, + other option is `PartitionsRoutingMode.UseSinglePartition` + * `lazy_start_partitioned_producers`: + This config affects producers of partitioned topics only. It controls whether + producers register and connect immediately to the owner broker of each partition + or start lazily on demand. The internal producer of one partition is always + started eagerly, chosen by the routing policy, but the internal producers of + any additional partitions are started on demand, upon receiving their first + message. + Using this mode can reduce the strain on brokers for topics with large numbers of + partitions and when the SinglePartition routing policy is used without keyed messages. + Because producer connection can be on demand, this can produce extra send latency + for the first messages of a given partition. + * `properties`: + Sets the properties for the producer. The properties associated with a producer + can be used for identify a producer at broker side. + * `batching_type`: + Sets the batching type for the producer. + There are two batching type: DefaultBatching and KeyBasedBatching. + - Default batching + incoming single messages: + (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + batched into single batch message: + [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] + + - KeyBasedBatching + incoming single messages: + (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + batched into single batch message: + [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] + * encryption_key: + The key used for symmetric encryption, configured on the producer side + * crypto_key_reader: + Symmetric encryption class implementation, configuring public key encryption messages for the producer + and private key decryption messages for the consumer + """ + _check_type(str, topic, 'topic') + _check_type_or_none(str, producer_name, 'producer_name') + _check_type(_schema.Schema, schema, 'schema') + _check_type_or_none(int, initial_sequence_id, 'initial_sequence_id') + _check_type(int, send_timeout_millis, 'send_timeout_millis') + _check_type(CompressionType, compression_type, 'compression_type') + _check_type(int, max_pending_messages, 'max_pending_messages') + _check_type(int, max_pending_messages_across_partitions, 'max_pending_messages_across_partitions') + _check_type(bool, block_if_queue_full, 'block_if_queue_full') + _check_type(bool, batching_enabled, 'batching_enabled') + _check_type(int, batching_max_messages, 'batching_max_messages') + _check_type(int, batching_max_allowed_size_in_bytes, 'batching_max_allowed_size_in_bytes') + _check_type(int, batching_max_publish_delay_ms, 'batching_max_publish_delay_ms') + _check_type_or_none(dict, properties, 'properties') + _check_type(BatchingType, batching_type, 'batching_type') + _check_type_or_none(str, encryption_key, 'encryption_key') + _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') + _check_type(bool, lazy_start_partitioned_producers, 'lazy_start_partitioned_producers') + + conf = _pulsar.ProducerConfiguration() + conf.send_timeout_millis(send_timeout_millis) + conf.compression_type(compression_type) + conf.max_pending_messages(max_pending_messages) + conf.max_pending_messages_across_partitions(max_pending_messages_across_partitions) + conf.block_if_queue_full(block_if_queue_full) + conf.batching_enabled(batching_enabled) + conf.batching_max_messages(batching_max_messages) + conf.batching_max_allowed_size_in_bytes(batching_max_allowed_size_in_bytes) + conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) + conf.partitions_routing_mode(message_routing_mode) + conf.batching_type(batching_type) + conf.lazy_start_partitioned_producers(lazy_start_partitioned_producers) + if producer_name: + conf.producer_name(producer_name) + if initial_sequence_id: + conf.initial_sequence_id(initial_sequence_id) + if properties: + for k, v in properties.items(): + conf.property(k, v) + + conf.schema(schema.schema_info()) + if encryption_key: + conf.encryption_key(encryption_key) + if crypto_key_reader: + conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) + + p = Producer() + p._producer = self._client.create_producer(topic, conf) + p._schema = schema + p._client = self._client + return p + + def subscribe(self, topic, subscription_name, + consumer_type=ConsumerType.Exclusive, + schema=schema.BytesSchema(), + message_listener=None, + receiver_queue_size=1000, + max_total_receiver_queue_size_across_partitions=50000, + consumer_name=None, + unacked_messages_timeout_ms=None, + broker_consumer_stats_cache_time_ms=30000, + negative_ack_redelivery_delay_ms=60000, + is_read_compacted=False, + properties=None, + pattern_auto_discovery_period=60, + initial_position=InitialPosition.Latest, + crypto_key_reader=None, + replicate_subscription_state_enabled=False + ): + """ + Subscribe to the given topic and subscription combination. + + **Args** + + * `topic`: The name of the topic, list of topics or regex pattern. + This method will accept these forms: + - `topic='my-topic'` + - `topic=['topic-1', 'topic-2', 'topic-3']` + - `topic=re.compile('persistent://public/default/topic-*')` + * `subscription`: The name of the subscription. + + **Options** + + * `consumer_type`: + Select the subscription type to be used when subscribing to the topic. + * `schema`: + Define the schema of the data that will be received by this consumer. + * `message_listener`: + Sets a message listener for the consumer. When the listener is set, + the application will receive messages through it. Calls to + `consumer.receive()` will not be allowed. The listener function needs + to accept (consumer, message), for example: + + #!python + def my_listener(consumer, message): + # process message + consumer.acknowledge(message) + + * `receiver_queue_size`: + Sets the size of the consumer receive queue. The consumer receive + queue controls how many messages can be accumulated by the consumer + before the application calls `receive()`. Using a higher value could + potentially increase the consumer throughput at the expense of higher + memory utilization. Setting the consumer queue size to zero decreases + the throughput of the consumer by disabling pre-fetching of messages. + This approach improves the message distribution on shared subscription + by pushing messages only to those consumers that are ready to process + them. Neither receive with timeout nor partitioned topics can be used + if the consumer queue size is zero. The `receive()` function call + should not be interrupted when the consumer queue size is zero. The + default value is 1000 messages and should work well for most use + cases. + * `max_total_receiver_queue_size_across_partitions` + Set the max total receiver queue size across partitions. + This setting will be used to reduce the receiver queue size for individual partitions + * `consumer_name`: + Sets the consumer name. + * `unacked_messages_timeout_ms`: + Sets the timeout in milliseconds for unacknowledged messages. The + timeout needs to be greater than 10 seconds. An exception is thrown if + the given value is less than 10 seconds. If a successful + acknowledgement is not sent within the timeout, all the unacknowledged + messages are redelivered. + * `negative_ack_redelivery_delay_ms`: + The delay after which to redeliver the messages that failed to be + processed (with the `consumer.negative_acknowledge()`) + * `broker_consumer_stats_cache_time_ms`: + Sets the time duration for which the broker-side consumer stats will + be cached in the client. + * `is_read_compacted`: + Selects whether to read the compacted version of the topic + * `properties`: + Sets the properties for the consumer. The properties associated with a consumer + can be used for identify a consumer at broker side. + * `pattern_auto_discovery_period`: + Periods of seconds for consumer to auto discover match topics. + * `initial_position`: + Set the initial position of a consumer when subscribing to the topic. + It could be either: `InitialPosition.Earliest` or `InitialPosition.Latest`. + Default: `Latest`. + * crypto_key_reader: + Symmetric encryption class implementation, configuring public key encryption messages for the producer + and private key decryption messages for the consumer + * replicate_subscription_state_enabled: + Set whether the subscription status should be replicated. + Default: `False`. + """ + _check_type(str, subscription_name, 'subscription_name') + _check_type(ConsumerType, consumer_type, 'consumer_type') + _check_type(_schema.Schema, schema, 'schema') + _check_type(int, receiver_queue_size, 'receiver_queue_size') + _check_type(int, max_total_receiver_queue_size_across_partitions, + 'max_total_receiver_queue_size_across_partitions') + _check_type_or_none(str, consumer_name, 'consumer_name') + _check_type_or_none(int, unacked_messages_timeout_ms, 'unacked_messages_timeout_ms') + _check_type(int, broker_consumer_stats_cache_time_ms, 'broker_consumer_stats_cache_time_ms') + _check_type(int, negative_ack_redelivery_delay_ms, 'negative_ack_redelivery_delay_ms') + _check_type(int, pattern_auto_discovery_period, 'pattern_auto_discovery_period') + _check_type(bool, is_read_compacted, 'is_read_compacted') + _check_type_or_none(dict, properties, 'properties') + _check_type(InitialPosition, initial_position, 'initial_position') + _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') + + conf = _pulsar.ConsumerConfiguration() + conf.consumer_type(consumer_type) + conf.read_compacted(is_read_compacted) + if message_listener: + conf.message_listener(_listener_wrapper(message_listener, schema)) + conf.receiver_queue_size(receiver_queue_size) + conf.max_total_receiver_queue_size_across_partitions(max_total_receiver_queue_size_across_partitions) + if consumer_name: + conf.consumer_name(consumer_name) + if unacked_messages_timeout_ms: + conf.unacked_messages_timeout_ms(unacked_messages_timeout_ms) + + conf.negative_ack_redelivery_delay_ms(negative_ack_redelivery_delay_ms) + conf.broker_consumer_stats_cache_time_ms(broker_consumer_stats_cache_time_ms) + if properties: + for k, v in properties.items(): + conf.property(k, v) + conf.subscription_initial_position(initial_position) + + conf.schema(schema.schema_info()) + + if crypto_key_reader: + conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) + + conf.replicate_subscription_state_enabled(replicate_subscription_state_enabled) + + c = Consumer() + if isinstance(topic, str): + # Single topic + c._consumer = self._client.subscribe(topic, subscription_name, conf) + elif isinstance(topic, list): + # List of topics + c._consumer = self._client.subscribe_topics(topic, subscription_name, conf) + elif isinstance(topic, _retype): + # Regex pattern + c._consumer = self._client.subscribe_pattern(topic.pattern, subscription_name, conf) + else: + raise ValueError("Argument 'topic' is expected to be of a type between (str, list, re.pattern)") + + c._client = self + c._schema = schema + self._consumers.append(c) + return c + + def create_reader(self, topic, start_message_id, + schema=schema.BytesSchema(), + reader_listener=None, + receiver_queue_size=1000, + reader_name=None, + subscription_role_prefix=None, + is_read_compacted=False, + crypto_key_reader=None + ): + """ + Create a reader on a particular topic + + **Args** + + * `topic`: The name of the topic. + * `start_message_id`: The initial reader positioning is done by specifying a message id. + The options are: + * `MessageId.earliest`: Start reading from the earliest message available in the topic + * `MessageId.latest`: Start reading from the end topic, only getting messages published + after the reader was created + * `MessageId`: When passing a particular message id, the reader will position itself on + that specific position. The first message to be read will be the message next to the + specified messageId. Message id can be serialized into a string and deserialized + back into a `MessageId` object: + + # Serialize to string + s = msg.message_id().serialize() + + # Deserialize from string + msg_id = MessageId.deserialize(s) + + **Options** + + * `schema`: + Define the schema of the data that will be received by this reader. + * `reader_listener`: + Sets a message listener for the reader. When the listener is set, + the application will receive messages through it. Calls to + `reader.read_next()` will not be allowed. The listener function needs + to accept (reader, message), for example: + + def my_listener(reader, message): + # process message + pass + + * `receiver_queue_size`: + Sets the size of the reader receive queue. The reader receive + queue controls how many messages can be accumulated by the reader + before the application calls `read_next()`. Using a higher value could + potentially increase the reader throughput at the expense of higher + memory utilization. + * `reader_name`: + Sets the reader name. + * `subscription_role_prefix`: + Sets the subscription role prefix. + * `is_read_compacted`: + Selects whether to read the compacted version of the topic + * crypto_key_reader: + Symmetric encryption class implementation, configuring public key encryption messages for the producer + and private key decryption messages for the consumer + """ + _check_type(str, topic, 'topic') + _check_type(_pulsar.MessageId, start_message_id, 'start_message_id') + _check_type(_schema.Schema, schema, 'schema') + _check_type(int, receiver_queue_size, 'receiver_queue_size') + _check_type_or_none(str, reader_name, 'reader_name') + _check_type_or_none(str, subscription_role_prefix, 'subscription_role_prefix') + _check_type(bool, is_read_compacted, 'is_read_compacted') + _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') + + conf = _pulsar.ReaderConfiguration() + if reader_listener: + conf.reader_listener(_listener_wrapper(reader_listener, schema)) + conf.receiver_queue_size(receiver_queue_size) + if reader_name: + conf.reader_name(reader_name) + if subscription_role_prefix: + conf.subscription_role_prefix(subscription_role_prefix) + conf.schema(schema.schema_info()) + conf.read_compacted(is_read_compacted) + if crypto_key_reader: + conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) + + c = Reader() + c._reader = self._client.create_reader(topic, start_message_id, conf) + c._client = self + c._schema = schema + self._consumers.append(c) + return c + + def get_topic_partitions(self, topic): + """ + Get the list of partitions for a given topic. + + If the topic is partitioned, this will return a list of partition names. If the topic is not + partitioned, the returned list will contain the topic name itself. + + This can be used to discover the partitions and create Reader, Consumer or Producer + instances directly on a particular partition. + :param topic: the topic name to lookup + :return: a list of partition name + """ + _check_type(str, topic, 'topic') + return self._client.get_topic_partitions(topic) + + def shutdown(self): + """ + Perform immediate shutdown of Pulsar client. + + Release all resources and close all producer, consumer, and readers without waiting + for ongoing operations to complete. + """ + self._client.shutdown() + + def close(self): + """ + Close the client and all the associated producers and consumers + """ + self._client.close() + + +class Producer: + """ + The Pulsar message producer, used to publish messages on a topic. + """ + + def topic(self): + """ + Return the topic which producer is publishing to + """ + return self._producer.topic() + + def producer_name(self): + """ + Return the producer name which could have been assigned by the + system or specified by the client + """ + return self._producer.producer_name() + + def last_sequence_id(self): + """ + Get the last sequence id that was published by this producer. + + This represent either the automatically assigned or custom sequence id + (set on the `MessageBuilder`) that was published and acknowledged by the broker. + + After recreating a producer with the same producer name, this will return the + last message that was published in the previous producer session, or -1 if + there no message was ever published. + """ + return self._producer.last_sequence_id() + + def send(self, content, + properties=None, + partition_key=None, + sequence_id=None, + replication_clusters=None, + disable_replication=False, + event_timestamp=None, + deliver_at=None, + deliver_after=None, + ): + """ + Publish a message on the topic. Blocks until the message is acknowledged + + Returns a `MessageId` object that represents where the message is persisted. + + **Args** + + * `content`: + A `bytes` object with the message payload. + + **Options** + + * `properties`: + A dict of application-defined string properties. + * `partition_key`: + Sets the partition key for message routing. A hash of this key is used + to determine the message's topic partition. + * `sequence_id`: + Specify a custom sequence id for the message being published. + * `replication_clusters`: + Override namespace replication clusters. Note that it is the caller's + responsibility to provide valid cluster names and that all clusters + have been previously configured as topics. Given an empty list, + the message will replicate according to the namespace configuration. + * `disable_replication`: + Do not replicate this message. + * `event_timestamp`: + Timestamp in millis of the timestamp of event creation + * `deliver_at`: + Specify the this message should not be delivered earlier than the + specified timestamp. + The timestamp is milliseconds and based on UTC + * `deliver_after`: + Specify a delay in timedelta for the delivery of the messages. + + """ + msg = self._build_msg(content, properties, partition_key, sequence_id, + replication_clusters, disable_replication, event_timestamp, + deliver_at, deliver_after) + return MessageId.deserialize(self._producer.send(msg)) + + def send_async(self, content, callback, + properties=None, + partition_key=None, + sequence_id=None, + replication_clusters=None, + disable_replication=False, + event_timestamp=None, + deliver_at=None, + deliver_after=None, + ): + """ + Send a message asynchronously. + + The `callback` will be invoked once the message has been acknowledged + by the broker. + + Example: + + #!python + def callback(res, msg_id): + print('Message published: %s' % res) + + producer.send_async(msg, callback) + + When the producer queue is full, by default the message will be rejected + and the callback invoked with an error code. + + **Args** + + * `content`: + A `bytes` object with the message payload. + + **Options** + + * `properties`: + A dict of application0-defined string properties. + * `partition_key`: + Sets the partition key for the message routing. A hash of this key is + used to determine the message's topic partition. + * `sequence_id`: + Specify a custom sequence id for the message being published. + * `replication_clusters`: Override namespace replication clusters. Note + that it is the caller's responsibility to provide valid cluster names + and that all clusters have been previously configured as topics. + Given an empty list, the message will replicate per the namespace + configuration. + * `disable_replication`: + Do not replicate this message. + * `event_timestamp`: + Timestamp in millis of the timestamp of event creation + * `deliver_at`: + Specify the this message should not be delivered earlier than the + specified timestamp. + The timestamp is milliseconds and based on UTC + * `deliver_after`: + Specify a delay in timedelta for the delivery of the messages. + """ + msg = self._build_msg(content, properties, partition_key, sequence_id, + replication_clusters, disable_replication, event_timestamp, + deliver_at, deliver_after) + self._producer.send_async(msg, callback) + + + def flush(self): + """ + Flush all the messages buffered in the client and wait until all messages have been + successfully persisted + """ + self._producer.flush() + + + def close(self): + """ + Close the producer. + """ + self._producer.close() + + def _build_msg(self, content, properties, partition_key, sequence_id, + replication_clusters, disable_replication, event_timestamp, + deliver_at, deliver_after): + data = self._schema.encode(content) + + _check_type(bytes, data, 'data') + _check_type_or_none(dict, properties, 'properties') + _check_type_or_none(str, partition_key, 'partition_key') + _check_type_or_none(int, sequence_id, 'sequence_id') + _check_type_or_none(list, replication_clusters, 'replication_clusters') + _check_type(bool, disable_replication, 'disable_replication') + _check_type_or_none(int, event_timestamp, 'event_timestamp') + _check_type_or_none(int, deliver_at, 'deliver_at') + _check_type_or_none(timedelta, deliver_after, 'deliver_after') + + mb = _pulsar.MessageBuilder() + mb.content(data) + if properties: + for k, v in properties.items(): + mb.property(k, v) + if partition_key: + mb.partition_key(partition_key) + if sequence_id: + mb.sequence_id(sequence_id) + if replication_clusters: + mb.replication_clusters(replication_clusters) + if disable_replication: + mb.disable_replication(disable_replication) + if event_timestamp: + mb.event_timestamp(event_timestamp) + if deliver_at: + mb.deliver_at(deliver_at) + if deliver_after: + mb.deliver_after(deliver_after) + + return mb.build() + + def is_connected(self): + """ + Check if the producer is connected or not. + """ + return self._producer.is_connected() + + +class Consumer: + """ + Pulsar consumer. + """ + + def topic(self): + """ + Return the topic this consumer is subscribed to. + """ + return self._consumer.topic() + + def subscription_name(self): + """ + Return the subscription name. + """ + return self._consumer.subscription_name() + + def unsubscribe(self): + """ + Unsubscribe the current consumer from the topic. + + This method will block until the operation is completed. Once the + consumer is unsubscribed, no more messages will be received and + subsequent new messages will not be retained for this consumer. + + This consumer object cannot be reused. + """ + return self._consumer.unsubscribe() + + def receive(self, timeout_millis=None): + """ + Receive a single message. + + If a message is not immediately available, this method will block until + a new message is available. + + **Options** + + * `timeout_millis`: + If specified, the receive will raise an exception if a message is not + available within the timeout. + """ + if timeout_millis is None: + msg = self._consumer.receive() + else: + _check_type(int, timeout_millis, 'timeout_millis') + msg = self._consumer.receive(timeout_millis) + + m = Message() + m._message = msg + m._schema = self._schema + return m + + def acknowledge(self, message): + """ + Acknowledge the reception of a single message. + + This method will block until an acknowledgement is sent to the broker. + After that, the message will not be re-delivered to this consumer. + + **Args** + + * `message`: + The received message or message id. + """ + if isinstance(message, Message): + self._consumer.acknowledge(message._message) + else: + self._consumer.acknowledge(message) + + def acknowledge_cumulative(self, message): + """ + Acknowledge the reception of all the messages in the stream up to (and + including) the provided message. + + This method will block until an acknowledgement is sent to the broker. + After that, the messages will not be re-delivered to this consumer. + + **Args** + + * `message`: + The received message or message id. + """ + if isinstance(message, Message): + self._consumer.acknowledge_cumulative(message._message) + else: + self._consumer.acknowledge_cumulative(message) + + def negative_acknowledge(self, message): + """ + Acknowledge the failure to process a single message. + + When a message is "negatively acked" it will be marked for redelivery after + some fixed delay. The delay is configurable when constructing the consumer + with {@link ConsumerConfiguration#setNegativeAckRedeliveryDelayMs}. + + This call is not blocking. + + **Args** + + * `message`: + The received message or message id. + """ + if isinstance(message, Message): + self._consumer.negative_acknowledge(message._message) + else: + self._consumer.negative_acknowledge(message) + + def pause_message_listener(self): + """ + Pause receiving messages via the `message_listener` until + `resume_message_listener()` is called. + """ + self._consumer.pause_message_listener() + + def resume_message_listener(self): + """ + Resume receiving the messages via the message listener. + Asynchronously receive all the messages enqueued from the time + `pause_message_listener()` was called. + """ + self._consumer.resume_message_listener() + + def redeliver_unacknowledged_messages(self): + """ + Redelivers all the unacknowledged messages. In failover mode, the + request is ignored if the consumer is not active for the given topic. In + shared mode, the consumer's messages to be redelivered are distributed + across all the connected consumers. This is a non-blocking call and + doesn't throw an exception. In case the connection breaks, the messages + are redelivered after reconnect. + """ + self._consumer.redeliver_unacknowledged_messages() + + def seek(self, messageid): + """ + Reset the subscription associated with this consumer to a specific message id or publish timestamp. + The message id can either be a specific message or represent the first or last messages in the topic. + Note: this operation can only be done on non-partitioned topics. For these, one can rather perform the + seek() on the individual partitions. + + **Args** + + * `message`: + The message id for seek, OR an integer event time to seek to + """ + self._consumer.seek(messageid) + + def close(self): + """ + Close the consumer. + """ + self._consumer.close() + self._client._consumers.remove(self) + + def is_connected(self): + """ + Check if the consumer is connected or not. + """ + return self._consumer.is_connected() + + + +class Reader: + """ + Pulsar topic reader. + """ + + def topic(self): + """ + Return the topic this reader is reading from. + """ + return self._reader.topic() + + def read_next(self, timeout_millis=None): + """ + Read a single message. + + If a message is not immediately available, this method will block until + a new message is available. + + **Options** + + * `timeout_millis`: + If specified, the receive will raise an exception if a message is not + available within the timeout. + """ + if timeout_millis is None: + msg = self._reader.read_next() + else: + _check_type(int, timeout_millis, 'timeout_millis') + msg = self._reader.read_next(timeout_millis) + + m = Message() + m._message = msg + m._schema = self._schema + return m + + def has_message_available(self): + """ + Check if there is any message available to read from the current position. + """ + return self._reader.has_message_available(); + + def seek(self, messageid): + """ + Reset this reader to a specific message id or publish timestamp. + The message id can either be a specific message or represent the first or last messages in the topic. + Note: this operation can only be done on non-partitioned topics. For these, one can rather perform the + seek() on the individual partitions. + + **Args** + + * `message`: + The message id for seek, OR an integer event time to seek to + """ + self._reader.seek(messageid) + + def close(self): + """ + Close the reader. + """ + self._reader.close() + self._client._consumers.remove(self) + + def is_connected(self): + """ + Check if the reader is connected or not. + """ + return self._reader.is_connected() + + +class CryptoKeyReader: + """ + Default crypto key reader implementation + """ + def __init__(self, public_key_path, private_key_path): + """ + Create crypto key reader. + + **Args** + + * `public_key_path`: Path to the public key + * `private_key_path`: Path to private key + """ + _check_type(str, public_key_path, 'public_key_path') + _check_type(str, private_key_path, 'private_key_path') + self.cryptoKeyReader = _pulsar.CryptoKeyReader(public_key_path, private_key_path) + +def _check_type(var_type, var, name): + if not isinstance(var, var_type): + raise ValueError("Argument %s is expected to be of type '%s' and not '%s'" + % (name, var_type.__name__, type(var).__name__)) + + +def _check_type_or_none(var_type, var, name): + if var is not None and not isinstance(var, var_type): + raise ValueError("Argument %s is expected to be either None or of type '%s'" + % (name, var_type.__name__)) + + +def _listener_wrapper(listener, schema): + def wrapper(consumer, msg): + c = Consumer() + c._consumer = consumer + m = Message() + m._message = msg + m._schema = schema + listener(c, m) + return wrapper diff --git a/conf/instances/python-instance/pulsar/exceptions.py b/conf/instances/python-instance/pulsar/exceptions.py new file mode 100644 index 0000000000000..d151564f90128 --- /dev/null +++ b/conf/instances/python-instance/pulsar/exceptions.py @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from _pulsar import PulsarException, UnknownError, InvalidConfiguration, Timeout, LookupError, ConnectError, \ + ReadError, AuthenticationError, AuthorizationError, ErrorGettingAuthenticationData, BrokerMetadataError, \ + BrokerPersistenceError, ChecksumError, ConsumerBusy, NotConnected, AlreadyClosed, InvalidMessage, \ + ConsumerNotInitialized, ProducerNotInitialized, ProducerBusy, TooManyLookupRequestException, InvalidTopicName, \ + InvalidUrl, ServiceUnitNotReady, OperationNotSupported, ProducerBlockedQuotaExceededError, \ + ProducerBlockedQuotaExceededException, ProducerQueueIsFull, MessageTooBig, TopicNotFound, SubscriptionNotFound, \ + ConsumerNotFound, UnsupportedVersionError, TopicTerminated, CryptoError, IncompatibleSchema, ConsumerAssignError, \ + CumulativeAcknowledgementNotAllowedError, TransactionCoordinatorNotFoundError, InvalidTxnStatusError, \ + NotAllowedError, TransactionConflict, TransactionNotFound, ProducerFenced, MemoryBufferIsFull diff --git a/conf/instances/python-instance/pulsar/functions/__init__.py b/conf/instances/python-instance/pulsar/functions/__init__.py new file mode 100644 index 0000000000000..b14a93244ff08 --- /dev/null +++ b/conf/instances/python-instance/pulsar/functions/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- diff --git a/conf/instances/python-instance/pulsar/functions/context.py b/conf/instances/python-instance/pulsar/functions/context.py new file mode 100644 index 0000000000000..fa9647913cb25 --- /dev/null +++ b/conf/instances/python-instance/pulsar/functions/context.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""context.py: Context defines context information available during +# processing of a request. +""" +from abc import abstractmethod + +class Context(object): + """Interface defining information available at process time""" + @abstractmethod + def get_message_id(self): + """Return the messageid of the current message that we are processing""" + pass + + @abstractmethod + def get_message_key(self): + """Return the key of the current message that we are processing""" + pass + + @abstractmethod + def get_message_eventtime(self): + """Return the event time of the current message that we are processing""" + pass + + @abstractmethod + def get_message_properties(self): + """Return the message properties kv map of the current message that we are processing""" + pass + + @abstractmethod + def get_current_message_topic_name(self): + """Returns the topic name of the message that we are processing""" + pass + + @abstractmethod + def get_function_tenant(self): + """Returns the tenant of the message that's being processed""" + pass + + @abstractmethod + def get_function_namespace(self): + """Returns the namespace of the message that's being processed""" + + @abstractmethod + def get_function_name(self): + """Returns the function name that we are a part of""" + pass + + @abstractmethod + def get_function_id(self): + """Returns the function id that we are a part of""" + pass + + @abstractmethod + def get_instance_id(self): + """Returns the instance id that is executing the function""" + pass + + @abstractmethod + def get_function_version(self): + """Returns the version of function that we are executing""" + pass + + @abstractmethod + def get_logger(self): + """Returns the logger object that can be used to do logging""" + pass + + @abstractmethod + def get_user_config_value(self, key): + """Returns the value of the user-defined config. If the key doesn't exist, None is returned""" + pass + + @abstractmethod + def get_user_config_map(self): + """Returns the entire user-defined config as a dict (the dict will be empty if no user-defined config is supplied)""" + pass + + @abstractmethod + def get_secret(self, secret_name): + """Returns the secret value associated with the name. None if nothing was found""" + pass + + @abstractmethod + def get_partition_key(self): + """Returns partition key of the input message is one exists""" + pass + + + @abstractmethod + def record_metric(self, metric_name, metric_value): + """Records the metric_value. metric_value has to satisfy isinstance(metric_value, numbers.Number)""" + pass + + @abstractmethod + def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None, message_conf=None): + """Publishes message to topic_name by first serializing the message using serde_class_name serde + The message will have properties specified if any + + The available options for message_conf: + + properties, + partition_key, + sequence_id, + replication_clusters, + disable_replication, + event_timestamp + + """ + pass + + @abstractmethod + def get_input_topics(self): + """Returns the input topics of function""" + pass + + @abstractmethod + def get_output_topic(self): + """Returns the output topic of function""" + pass + + @abstractmethod + def get_output_serde_class_name(self): + """return output Serde class""" + pass + + @abstractmethod + def ack(self, msgid, topic): + """ack this message id""" + pass + + @abstractmethod + def incr_counter(self, key, amount): + """incr the counter of a given key in the managed state""" + pass + + @abstractmethod + def get_counter(self, key): + """get the counter of a given key in the managed state""" + pass + + @abstractmethod + def del_counter(self, key): + """delete the counter of a given key in the managed state""" + pass + + @abstractmethod + def put_state(self, key, value): + """update the value of a given key in the managed state""" + pass + + @abstractmethod + def get_state(self, key): + """get the value of a given key in the managed state""" + pass diff --git a/conf/instances/python-instance/pulsar/functions/function.py b/conf/instances/python-instance/pulsar/functions/function.py new file mode 100644 index 0000000000000..dde92b8a55fec --- /dev/null +++ b/conf/instances/python-instance/pulsar/functions/function.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""function.py: This is the core interface of the function api. +# The process method is called for every message of the input topic of the +# function. The incoming input bytes are deserialized using the serde. +# The process function can optionally emit an output +""" +from abc import abstractmethod + +class Function(object): + """Interface for Pulsar Function""" + @abstractmethod + def process(self, input, context): + """Process input message""" + pass \ No newline at end of file diff --git a/conf/instances/python-instance/pulsar/functions/serde.py b/conf/instances/python-instance/pulsar/functions/serde.py new file mode 100644 index 0000000000000..968c1c9bc3aae --- /dev/null +++ b/conf/instances/python-instance/pulsar/functions/serde.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""serde.py: SerDe defines the interface for serialization/deserialization. +# Everytime a message is read from pulsar topic, the serde is invoked to +# serialize the bytes into an object before invoking the process method. +# Anytime a python object needs to be written back to pulsar, it is +# serialized into bytes before writing. +""" +from abc import abstractmethod + +import pickle + +class SerDe(object): + """Interface for Serialization/Deserialization""" + @abstractmethod + def serialize(self, input): + """Serialize input message into bytes""" + pass + + @abstractmethod + def deserialize(self, input_bytes): + """Serialize input_bytes into an object""" + pass + +class PickleSerDe(SerDe): + """Pickle based serializer""" + def serialize(self, input): + return pickle.dumps(input) + + def deserialize(self, input_bytes): + return pickle.loads(input_bytes) + +class IdentitySerDe(SerDe): + """Simple Serde that just conversion to string and back""" + def __init__(self): + self._types = [int, float, complex, str] + + def serialize(self, input): + if type(input) in self._types: + return str(input).encode('utf-8') + if type(input) == bytes: + return input + raise TypeError("IdentitySerde cannot serialize object of type %s" % type(input)) + + def deserialize(self, input_bytes): + for typ in self._types: + try: + return typ(input_bytes.decode('utf-8')) + except: + pass + return input_bytes diff --git a/conf/instances/python-instance/pulsar/schema/__init__.py b/conf/instances/python-instance/pulsar/schema/__init__.py new file mode 100644 index 0000000000000..efa680666a729 --- /dev/null +++ b/conf/instances/python-instance/pulsar/schema/__init__.py @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from .definition import Record, Field, Null, Boolean, Integer, Long, \ + Float, Double, Bytes, String, Array, Map, CustomEnum + +from .schema import Schema, BytesSchema, StringSchema, JsonSchema +from .schema_avro import AvroSchema diff --git a/conf/instances/python-instance/pulsar/schema/definition.py b/conf/instances/python-instance/pulsar/schema/definition.py new file mode 100644 index 0000000000000..a7a235b25a6e3 --- /dev/null +++ b/conf/instances/python-instance/pulsar/schema/definition.py @@ -0,0 +1,516 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import copy +from abc import abstractmethod +from collections import OrderedDict +from enum import Enum, EnumMeta +from six import with_metaclass + + +def _check_record_or_field(x): + if (type(x) is type and not issubclass(x, Record)) \ + and not isinstance(x, Field): + raise Exception('Argument ' + x + ' is not a Record or a Field') + + +class RecordMeta(type): + def __new__(metacls, name, parents, dct): + if name != 'Record': + # Do not apply this logic to the base class itself + dct['_fields'] = RecordMeta._get_fields(dct) + dct['_required'] = False + return type.__new__(metacls, name, parents, dct) + + @classmethod + def _get_fields(cls, dct): + # Build a set of valid fields for this record + fields = OrderedDict() + for name, value in dct.items(): + if issubclass(type(value), EnumMeta): + value = CustomEnum(value) + elif type(value) == RecordMeta: + # We expect an instance of a record rather than the class itself + value = value() + + if isinstance(value, Record) or isinstance(value, Field): + fields[name] = value + return fields + + +class Record(with_metaclass(RecordMeta, object)): + + # This field is used to set namespace for Avro Record schema. + _avro_namespace = None + + # Generate a schema where fields are sorted alphabetically + _sorted_fields = False + + def __init__(self, default=None, required_default=False, required=False, *args, **kwargs): + self._required_default = required_default + self._default = default + self._required = required + + for k, value in self._fields.items(): + if k in kwargs: + if isinstance(value, Record) and isinstance(kwargs[k], dict): + # Use dict init Record object + copied = copy.copy(value) + copied.__init__(**kwargs[k]) + self.__setattr__(k, copied) + elif isinstance(value, Array) and isinstance(kwargs[k], list) and len(kwargs[k]) > 0 \ + and isinstance(value.array_type, Record) and isinstance(kwargs[k][0], dict): + arr = [] + for item in kwargs[k]: + copied = copy.copy(value.array_type) + copied.__init__(**item) + arr.append(copied) + self.__setattr__(k, arr) + elif isinstance(value, Map) and isinstance(kwargs[k], dict) and len(kwargs[k]) > 0 \ + and isinstance(value.value_type, Record) and isinstance(list(kwargs[k].values())[0], dict): + dic = {} + for mapKey, mapValue in kwargs[k].items(): + copied = copy.copy(value.value_type) + copied.__init__(**mapValue) + dic[mapKey] = copied + self.__setattr__(k, dic) + else: + # Value was overridden at constructor + self.__setattr__(k, kwargs[k]) + elif isinstance(value, Record): + # Value is a subrecord + self.__setattr__(k, value) + else: + # Set field to default value, without revalidating the default value type + super(Record, self).__setattr__(k, value.default()) + + @classmethod + def schema(cls): + return cls.schema_info(set()) + + @classmethod + def schema_info(cls, defined_names): + namespace_prefix = '' + if cls._avro_namespace is not None: + namespace_prefix = cls._avro_namespace + '.' + namespace_name = namespace_prefix + cls.__name__ + + if namespace_name in defined_names: + return namespace_name + + defined_names.add(namespace_name) + + schema = { + 'type': 'record', + 'name': str(cls.__name__) + } + if cls._avro_namespace is not None: + schema['namespace'] = cls._avro_namespace + schema['fields'] = [] + + def get_filed_default_value(value): + if isinstance(value, Enum): + return value.name + else: + return value + + if cls._sorted_fields: + fields = sorted(cls._fields.keys()) + else: + fields = cls._fields.keys() + for name in fields: + field = cls._fields[name] + field_type = field.schema_info(defined_names) \ + if field._required else ['null', field.schema_info(defined_names)] + schema['fields'].append({ + 'name': name, + 'default': get_filed_default_value(field.default()), + 'type': field_type + }) if field.required_default() else schema['fields'].append({ + 'name': name, + 'type': field_type, + }) + + return schema + + def __setattr__(self, key, value): + if key == '_default': + super(Record, self).__setattr__(key, value) + elif key == '_required_default': + super(Record, self).__setattr__(key, value) + elif key == '_required': + super(Record, self).__setattr__(key, value) + else: + if key not in self._fields: + raise AttributeError('Cannot set undeclared field ' + key + ' on record') + + # Check that type of value matches the field type + field = self._fields[key] + value = field.validate_type(key, value) + super(Record, self).__setattr__(key, value) + + def __eq__(self, other): + for field in self._fields: + if self.__getattribute__(field) != other.__getattribute__(field): + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __str__(self): + return str(self.__dict__) + + def type(self): + return str(self.__class__.__name__) + + def python_type(self): + return self.__class__ + + def validate_type(self, name, val): + if val is None and not self._required: + return self.default() + + if not isinstance(val, self.__class__): + raise TypeError("Invalid type '%s' for sub-record field '%s'. Expected: %s" % ( + type(val), name, self.__class__)) + return val + + def default(self): + if self._default is not None: + return self._default + else: + return None + + def required_default(self): + return self._required_default + + +class Field(object): + def __init__(self, default=None, required=False, required_default=False): + if default is not None: + default = self.validate_type('default', default) + self._default = default + self._required_default = required_default + self._required = required + + @abstractmethod + def type(self): + pass + + @abstractmethod + def python_type(self): + pass + + def validate_type(self, name, val): + if val is None and not self._required: + return self.default() + + if type(val) != self.python_type(): + raise TypeError("Invalid type '%s' for field '%s'. Expected: %s" % (type(val), name, self.python_type())) + return val + + def schema(self): + # For primitive types, the schema would just be the type itself + return self.type() + + def schema_info(self, defined_names): + return self.type() + + def default(self): + return self._default + + def required_default(self): + return self._required_default + + +# All types + + +class Null(Field): + def type(self): + return 'null' + + def python_type(self): + return type(None) + + def validate_type(self, name, val): + if val is not None: + raise TypeError('Field ' + name + ' is set to be None') + return val + + +class Boolean(Field): + def type(self): + return 'boolean' + + def python_type(self): + return bool + + def default(self): + if self._default is not None: + return self._default + else: + return False + + +class Integer(Field): + def type(self): + return 'int' + + def python_type(self): + return int + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Long(Field): + def type(self): + return 'long' + + def python_type(self): + return int + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Float(Field): + def type(self): + return 'float' + + def python_type(self): + return float + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Double(Field): + def type(self): + return 'double' + + def python_type(self): + return float + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Bytes(Field): + def type(self): + return 'bytes' + + def python_type(self): + return bytes + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class String(Field): + def type(self): + return 'string' + + def python_type(self): + return str + + def validate_type(self, name, val): + t = type(val) + + if val is None and not self._required: + return self.default() + + if not (t is str or t.__name__ == 'unicode'): + raise TypeError("Invalid type '%s' for field '%s'. Expected a string" % (t, name)) + return val + + def default(self): + if self._default is not None: + return self._default + else: + return None + +# Complex types + + +class CustomEnum(Field): + def __init__(self, enum_type, default=None, required=False, required_default=False): + if not issubclass(enum_type, Enum): + raise Exception(enum_type + " is not a valid Enum type") + self.enum_type = enum_type + self.values = {} + for x in enum_type.__members__.values(): + self.values[x.value] = x + super(CustomEnum, self).__init__(default, required, required_default) + + def type(self): + return 'enum' + + def python_type(self): + return self.enum_type + + def validate_type(self, name, val): + if val is None: + return None + + if type(val) is str: + # The enum was passed as a string, we need to check it against the possible values + if val in self.enum_type.__members__: + return self.enum_type.__members__[val] + else: + raise TypeError( + "Invalid enum value '%s' for field '%s'. Expected: %s" % (val, name, self.enum_type.__members__.keys())) + elif type(val) is int: + # The enum was passed as an int, we need to check it against the possible values + if val in self.values: + return self.values[val] + else: + raise TypeError( + "Invalid enum value '%s' for field '%s'. Expected: %s" % (val, name, self.values.keys())) + elif type(val) != self.python_type(): + raise TypeError("Invalid type '%s' for field '%s'. Expected: %s" % (type(val), name, self.python_type())) + else: + return val + + def schema(self): + return self.schema_info(set()) + + def schema_info(self, defined_names): + if self.enum_type.__name__ in defined_names: + return self.enum_type.__name__ + defined_names.add(self.enum_type.__name__) + return { + 'type': self.type(), + 'name': self.enum_type.__name__, + 'symbols': [x.name for x in self.enum_type] + } + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Array(Field): + def __init__(self, array_type, default=None, required=False, required_default=False): + _check_record_or_field(array_type) + self.array_type = array_type + super(Array, self).__init__(default=default, required=required, required_default=required_default) + + def type(self): + return 'array' + + def python_type(self): + return list + + def validate_type(self, name, val): + if val is None: + return None + + super(Array, self).validate_type(name, val) + + for x in val: + if type(x) != self.array_type.python_type(): + raise TypeError('Array field ' + name + ' items should all be of type ' + + self.array_type.python_type()) + return val + + def schema(self): + return self.schema_info(set()) + + def schema_info(self, defined_names): + return { + 'type': self.type(), + 'items': self.array_type.schema_info(defined_names) if isinstance(self.array_type, (Array, Map, Record)) + else self.array_type.type() + } + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +class Map(Field): + def __init__(self, value_type, default=None, required=False, required_default=False): + _check_record_or_field(value_type) + self.value_type = value_type + super(Map, self).__init__(default=default, required=required, required_default=required_default) + + def type(self): + return 'map' + + def python_type(self): + return dict + + def validate_type(self, name, val): + if val is None: + return None + + super(Map, self).validate_type(name, val) + + for k, v in val.items(): + if type(k) != str and not is_unicode(k): + raise TypeError('Map keys for field ' + name + ' should all be strings') + if type(v) != self.value_type.python_type(): + raise TypeError('Map values for field ' + name + ' should all be of type ' + + self.value_type.python_type()) + + return val + + def schema(self): + return self.schema_info(set()) + + def schema_info(self, defined_names): + return { + 'type': self.type(), + 'values': self.value_type.schema_info(defined_names) if isinstance(self.value_type, (Array, Map, Record)) + else self.value_type.type() + } + + def default(self): + if self._default is not None: + return self._default + else: + return None + + +# Python3 has no `unicode` type, so here we use a tricky way to check if the type of `x` is `unicode` in Python2 +# and also make it work well with Python3. +def is_unicode(x): + return 'encode' in dir(x) and type(x.encode()) == str diff --git a/conf/instances/python-instance/pulsar/schema/schema.py b/conf/instances/python-instance/pulsar/schema/schema.py new file mode 100644 index 0000000000000..faf2c89ae2587 --- /dev/null +++ b/conf/instances/python-instance/pulsar/schema/schema.py @@ -0,0 +1,107 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +from abc import abstractmethod +import json +import _pulsar +import enum + + +class Schema(object): + def __init__(self, record_cls, schema_type, schema_definition, schema_name): + self._record_cls = record_cls + self._schema_info = _pulsar.SchemaInfo(schema_type, schema_name, + json.dumps(schema_definition, indent=True)) + + @abstractmethod + def encode(self, obj): + pass + + @abstractmethod + def decode(self, data): + pass + + def schema_info(self): + return self._schema_info + + def _validate_object_type(self, obj): + if not isinstance(obj, self._record_cls): + raise TypeError('Invalid record obj of type ' + str(type(obj)) + + ' - expected type is ' + str(self._record_cls)) + + +class BytesSchema(Schema): + def __init__(self): + super(BytesSchema, self).__init__(bytes, _pulsar.SchemaType.BYTES, None, 'BYTES') + + def encode(self, data): + self._validate_object_type(data) + return data + + def decode(self, data): + return data + + def __str__(self): + return 'BytesSchema' + + +class StringSchema(Schema): + def __init__(self): + super(StringSchema, self).__init__(str, _pulsar.SchemaType.STRING, None, 'STRING') + + def encode(self, obj): + self._validate_object_type(obj) + return obj.encode('utf-8') + + def decode(self, data): + return data.decode('utf-8') + + def __str__(self): + return 'StringSchema' + + + +class JsonSchema(Schema): + + def __init__(self, record_cls): + super(JsonSchema, self).__init__(record_cls, _pulsar.SchemaType.JSON, + record_cls.schema(), 'JSON') + + def _get_serialized_value(self, o): + if isinstance(o, enum.Enum): + return o.value + else: + return o.__dict__ + + def encode(self, obj): + self._validate_object_type(obj) + # Copy the dict of the object as to not modify the provided object via the reference provided + data = obj.__dict__.copy() + if '_default' in data: + del data['_default'] + if '_required' in data: + del data['_required'] + if '_required_default' in data: + del data['_required_default'] + + return json.dumps(data, default=self._get_serialized_value, indent=True).encode('utf-8') + + def decode(self, data): + return self._record_cls(**json.loads(data)) diff --git a/conf/instances/python-instance/pulsar/schema/schema_avro.py b/conf/instances/python-instance/pulsar/schema/schema_avro.py new file mode 100644 index 0000000000000..05ceb8e20eff3 --- /dev/null +++ b/conf/instances/python-instance/pulsar/schema/schema_avro.py @@ -0,0 +1,96 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import _pulsar +import io +import enum + +from . import Record +from .schema import Schema + +try: + import fastavro + HAS_AVRO = True +except ImportError: + HAS_AVRO = False + +if HAS_AVRO: + class AvroSchema(Schema): + def __init__(self, record_cls, schema_definition=None): + if record_cls is None and schema_definition is None: + raise AssertionError("The param record_cls and schema_definition shouldn't be both None.") + + if record_cls is not None: + self._schema = record_cls.schema() + else: + self._schema = schema_definition + super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, self._schema, 'AVRO') + + def _get_serialized_value(self, x): + if isinstance(x, enum.Enum): + return x.name + elif isinstance(x, Record): + return self.encode_dict(x.__dict__) + elif isinstance(x, list): + arr = [] + for item in x: + arr.append(self._get_serialized_value(item)) + return arr + elif isinstance(x, dict): + return self.encode_dict(x) + else: + return x + + def encode(self, obj): + buffer = io.BytesIO() + m = obj + if self._record_cls is not None: + self._validate_object_type(obj) + m = self.encode_dict(obj.__dict__) + elif not isinstance(obj, dict): + raise ValueError('If using the custom schema, the record data should be dict type.') + + fastavro.schemaless_writer(buffer, self._schema, m) + return buffer.getvalue() + + def encode_dict(self, d): + obj = {} + for k, v in d.items(): + obj[k] = self._get_serialized_value(v) + return obj + + def decode(self, data): + buffer = io.BytesIO(data) + d = fastavro.schemaless_reader(buffer, self._schema) + if self._record_cls is not None: + return self._record_cls(**d) + else: + return d + +else: + class AvroSchema(Schema): + def __init__(self, _record_cls, _schema_definition): + raise Exception("Avro library support was not found. Make sure to install Pulsar client " + + "with Avro support: pip3 install 'pulsar-client[avro]'") + + def encode(self, obj): + pass + + def decode(self, data): + pass diff --git a/conf/instances/python-instance/python_instance.py b/conf/instances/python-instance/python_instance.py new file mode 100644 index 0000000000000..54e4a347c05c8 --- /dev/null +++ b/conf/instances/python-instance/python_instance.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""python_instance.py: Python Instance for running python functions +""" +import base64 +import os +import signal +import time +try: + import Queue as queue +except: + import queue +import threading +import sys +import re +import pulsar +import contextimpl +import Function_pb2 +import log +import util +import InstanceCommunication_pb2 + +# state dependencies +import state_context + +from functools import partial +from collections import namedtuple +from function_stats import Stats + +Log = log.Log +# Equivalent of the InstanceConfig in Java +InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_details max_buffered_tuples') +# This is the message that the consumers put on the queue for the function thread to process +InternalMessage = namedtuple('InternalMessage', 'message topic serde consumer') +InternalQuitMessage = namedtuple('InternalQuitMessage', 'quit') +DEFAULT_SERIALIZER = "serde.IdentitySerDe" + +PY3 = sys.version_info[0] >= 3 + +def base64ify(bytes_or_str): + if PY3 and isinstance(bytes_or_str, str): + input_bytes = bytes_or_str.encode('utf8') + else: + input_bytes = bytes_or_str + + output_bytes = base64.urlsafe_b64encode(input_bytes) + if PY3: + return output_bytes.decode('ascii') + else: + return output_bytes + +class PythonInstance(object): + def __init__(self, + instance_id, + function_id, + function_version, + function_details, + max_buffered_tuples, + expected_healthcheck_interval, + user_code, + pulsar_client, + secrets_provider, + cluster_name, + state_storage_serviceurl): + self.instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) + self.user_code = user_code + # set queue size to one since consumers already have internal queues. Just use queue to communicate message from + # consumers to processing thread + self.queue = queue.Queue(1) + self.log_topic_handler = None + if function_details.logTopic is not None and function_details.logTopic != "": + self.log_topic_handler = log.LogTopicHandler(str(function_details.logTopic), pulsar_client) + self.pulsar_client = pulsar_client + self.state_storage_serviceurl = state_storage_serviceurl + self.input_serdes = {} + self.consumers = {} + self.output_serde = None + self.function_class = None + self.function_purefunction = None + self.producer = None + self.execution_thread = None + self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATMOST_ONCE') + self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATLEAST_ONCE') + self.auto_ack = self.instance_config.function_details.autoAck + self.contextimpl = None + self.last_health_check_ts = time.time() + self.timeout_ms = function_details.source.timeoutMs if function_details.source.timeoutMs > 0 else None + self.expected_healthcheck_interval = expected_healthcheck_interval + self.secrets_provider = secrets_provider + self.state_context = state_context.NullStateContext() + self.metrics_labels = [function_details.tenant, + "%s/%s" % (function_details.tenant, function_details.namespace), + function_details.name, + instance_id, cluster_name, + "%s/%s/%s" % (function_details.tenant, function_details.namespace, function_details.name)] + self.stats = Stats(self.metrics_labels) + + def health_check(self): + self.last_health_check_ts = time.time() + health_check_result = InstanceCommunication_pb2.HealthCheckResult() + health_check_result.success = True + return health_check_result + + def process_spawner_health_check_timer(self): + if time.time() - self.last_health_check_ts > self.expected_healthcheck_interval * 3: + Log.critical("Haven't received health check from spawner in a while. Stopping instance...") + os.kill(os.getpid(), signal.SIGKILL) + sys.exit(1) + + def run(self): + # Setup state + self.state_context = self.setup_state() + + # Setup consumers and input deserializers + mode = pulsar._pulsar.ConsumerType.Shared + if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value("FAILOVER"): + mode = pulsar._pulsar.ConsumerType.Failover + + subscription_name = self.instance_config.function_details.source.subscriptionName + + if not (subscription_name and subscription_name.strip()): + subscription_name = str(self.instance_config.function_details.tenant) + "/" + \ + str(self.instance_config.function_details.namespace) + "/" + \ + str(self.instance_config.function_details.name) + + properties = util.get_properties(util.getFullyQualifiedFunctionName( + self.instance_config.function_details.tenant, + self.instance_config.function_details.namespace, + self.instance_config.function_details.name), + self.instance_config.instance_id) + + for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(): + if not serde: + serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER) + else: + serde_kclass = util.import_class(os.path.dirname(self.user_code), serde) + self.input_serdes[topic] = serde_kclass() + Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) + + self.consumers[topic] = self.pulsar_client.subscribe( + str(topic), subscription_name, + consumer_type=mode, + message_listener=partial(self.message_listener, self.input_serdes[topic]), + unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None, + properties=properties + ) + + for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items(): + if not consumer_conf.serdeClassName: + serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER) + else: + serde_kclass = util.import_class(os.path.dirname(self.user_code), consumer_conf.serdeClassName) + self.input_serdes[topic] = serde_kclass() + Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) + + consumer_args = { + "consumer_type": mode, + "message_listener": partial(self.message_listener, self.input_serdes[topic]), + "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, + "properties": properties + } + if consumer_conf.HasField("receiverQueueSize"): + consumer_args["receiver_queue_size"] = consumer_conf.receiverQueueSize.value + + if consumer_conf.isRegexPattern: + self.consumers[topic] = self.pulsar_client.subscribe( + re.compile(str(topic)), subscription_name, + **consumer_args + ) + else: + self.consumers[topic] = self.pulsar_client.subscribe( + str(topic), subscription_name, + **consumer_args + ) + + function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.className) + if function_kclass is None: + Log.critical("Could not import User Function Module %s" % self.instance_config.function_details.className) + raise NameError("Could not import User Function Module %s" % self.instance_config.function_details.className) + try: + self.function_class = function_kclass() + except: + self.function_purefunction = function_kclass + + self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log, self.pulsar_client, + self.user_code, self.consumers, + self.secrets_provider, self.metrics_labels, + self.state_context, self.stats) + # Now launch a thread that does execution + self.execution_thread = threading.Thread(target=self.actual_execution) + self.execution_thread.start() + + # start proccess spawner health check timer + self.last_health_check_ts = time.time() + if self.expected_healthcheck_interval > 0: + timer = util.FixedTimer(self.expected_healthcheck_interval, self.process_spawner_health_check_timer, name="health-check-timer") + timer.start() + + def actual_execution(self): + Log.debug("Started Thread for executing the function") + + while True: + try: + msg = self.queue.get(True) + if isinstance(msg, InternalQuitMessage): + break + Log.debug("Got a message from topic %s" % msg.topic) + # deserialize message + input_object = msg.serde.deserialize(msg.message.data()) + # set current message in context + self.contextimpl.set_current_message_context(msg.message, msg.topic) + output_object = None + self.saved_log_handler = None + if self.log_topic_handler is not None: + self.saved_log_handler = log.remove_all_handlers() + log.add_handler(self.log_topic_handler) + successfully_executed = False + try: + # get user function start time for statistic calculation + self.stats.set_last_invocation(time.time()) + + # start timer for process time + self.stats.process_time_start() + if self.function_class is not None: + output_object = self.function_class.process(input_object, self.contextimpl) + else: + output_object = self.function_purefunction.process(input_object) + successfully_executed = True + + # stop timer for process time + self.stats.process_time_end() + except Exception as e: + Log.exception("Exception while executing user method") + self.stats.incr_total_user_exceptions(e) + # If function throws exception then send neg ack for input message back to broker + msg.consumer.negative_acknowledge(msg.message) + + if self.log_topic_handler is not None: + log.remove_all_handlers() + log.add_handler(self.saved_log_handler) + if successfully_executed: + self.process_result(output_object, msg) + self.stats.incr_total_processed_successfully() + + except Exception as e: + Log.error("Uncaught exception in Python instance: %s" % e); + self.stats.incr_total_sys_exceptions(e) + if msg: + msg.consumer.negative_acknowledge(msg.message) + + def done_producing(self, consumer, orig_message, topic, result, sent_message): + if result == pulsar.Result.Ok: + if self.auto_ack: + consumer.acknowledge(orig_message) + else: + error_msg = "Failed to publish to topic [%s] with error [%s] with src message id [%s]" % (topic, result, orig_message.message_id()) + Log.error(error_msg) + self.stats.incr_total_sys_exceptions(Exception(error_msg)) + # If producer fails send output then send neg ack for input message back to broker + consumer.negative_acknowledge(orig_message) + + + def process_result(self, output, msg): + if output is not None and self.instance_config.function_details.sink.topic != None and \ + len(self.instance_config.function_details.sink.topic) > 0: + if self.output_serde is None: + self.setup_output_serde() + if self.producer is None: + self.setup_producer() + + # serialize function output + output_bytes = self.output_serde.serialize(output) + + if output_bytes is not None: + props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())} + self.producer.send_async(output_bytes, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) + elif self.auto_ack and self.atleast_once: + msg.consumer.acknowledge(msg.message) + + def setup_output_serde(self): + if self.instance_config.function_details.sink.serDeClassName != None and \ + len(self.instance_config.function_details.sink.serDeClassName) > 0: + serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.sink.serDeClassName) + self.output_serde = serde_kclass() + else: + global DEFAULT_SERIALIZER + serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER) + self.output_serde = serde_kclass() + + def setup_producer(self): + if self.instance_config.function_details.sink.topic != None and \ + len(self.instance_config.function_details.sink.topic) > 0: + Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic) + + batch_type = pulsar.BatchingType.Default + if self.instance_config.function_details.sink.producerSpec.batchBuilder != None and \ + len(self.instance_config.function_details.sink.producerSpec.batchBuilder) > 0: + batch_builder = self.instance_config.function_details.sink.producerSpec.batchBuilder + if batch_builder == "KEY_BASED": + batch_type = pulsar.BatchingType.KeyBased + + self.producer = self.pulsar_client.create_producer( + str(self.instance_config.function_details.sink.topic), + block_if_queue_full=True, + batching_enabled=True, + batching_type=batch_type, + batching_max_publish_delay_ms=10, + compression_type=pulsar.CompressionType.LZ4, + # set send timeout to be infinity to prevent potential deadlock with consumer + # that might happen when consumer is blocked due to unacked messages + send_timeout_millis=0, + properties=util.get_properties(util.getFullyQualifiedFunctionName( + self.instance_config.function_details.tenant, + self.instance_config.function_details.namespace, + self.instance_config.function_details.name), + self.instance_config.instance_id) + ) + + def setup_state(self): + table_ns = "%s_%s" % (str(self.instance_config.function_details.tenant), + str(self.instance_config.function_details.namespace)) + table_ns = table_ns.replace("-", "_") + table_name = str(self.instance_config.function_details.name) + return state_context.create_state_context(self.state_storage_serviceurl, table_ns, table_name) + + def message_listener(self, serde, consumer, message): + # increment number of received records from source + self.stats.incr_total_received() + item = InternalMessage(message, message.topic_name(), serde, consumer) + self.queue.put(item, True) + if self.atmost_once and self.auto_ack: + consumer.acknowledge(message) + + def get_and_reset_metrics(self): + # First get any user metrics + metrics = self.get_metrics() + self.reset_metrics() + return metrics + + def reset_metrics(self): + self.stats.reset() + self.contextimpl.reset_metrics() + + def get_metrics(self): + + total_received = self.stats.get_total_received() + total_processed_successfully = self.stats.get_total_processed_successfully() + total_user_exceptions = self.stats.get_total_user_exceptions() + total_sys_exceptions = self.stats.get_total_sys_exceptions() + avg_process_latency_ms = self.stats.get_avg_process_latency() + last_invocation = self.stats.get_last_invocation() + + total_received_1min = self.stats.get_total_received_1min() + total_processed_successfully_1min = self.stats.get_total_processed_successfully_1min() + total_user_exceptions_1min = self.stats.get_total_user_exceptions_1min() + total_sys_exceptions_1min = self.stats.get_total_sys_exceptions_1min() + avg_process_latency_ms_1min = self.stats.get_avg_process_latency_1min() + + metrics_data = InstanceCommunication_pb2.MetricsData() + # total metrics + metrics_data.receivedTotal = int(total_received) if sys.version_info.major >= 3 else long(total_received) + metrics_data.processedSuccessfullyTotal = int(total_processed_successfully) if sys.version_info.major >= 3 else long(total_processed_successfully) + metrics_data.systemExceptionsTotal = int(total_sys_exceptions) if sys.version_info.major >= 3 else long(total_sys_exceptions) + metrics_data.userExceptionsTotal = int(total_user_exceptions) if sys.version_info.major >= 3 else long(total_user_exceptions) + metrics_data.avgProcessLatency = avg_process_latency_ms + metrics_data.lastInvocation = int(last_invocation) if sys.version_info.major >= 3 else long(last_invocation) + # 1min metrics + metrics_data.receivedTotal_1min = int(total_received_1min) if sys.version_info.major >= 3 else long(total_received_1min) + metrics_data.processedSuccessfullyTotal_1min = int( + total_processed_successfully_1min) if sys.version_info.major >= 3 else long(total_processed_successfully_1min) + metrics_data.systemExceptionsTotal_1min = int(total_sys_exceptions_1min) if sys.version_info.major >= 3 else long( + total_sys_exceptions_1min) + metrics_data.userExceptionsTotal_1min = int(total_user_exceptions_1min) if sys.version_info.major >= 3 else long( + total_user_exceptions_1min) + metrics_data.avgProcessLatency_1min = avg_process_latency_ms_1min + + # get any user metrics + user_metrics = self.contextimpl.get_metrics() + for metric_name, value in user_metrics.items(): + metrics_data.userMetrics[metric_name] = value + + return metrics_data + + def add_system_metrics(self, metric_name, value, metrics): + metrics.metrics[metric_name].count = value + metrics.metrics[metric_name].sum = value + metrics.metrics[metric_name].min = 0 + metrics.metrics[metric_name].max = value + + def get_function_status(self): + status = InstanceCommunication_pb2.FunctionStatus() + status.running = True + + total_received = self.stats.get_total_received() + total_processed_successfully = self.stats.get_total_processed_successfully() + total_user_exceptions = self.stats.get_total_user_exceptions() + total_sys_exceptions = self.stats.get_total_sys_exceptions() + avg_process_latency_ms = self.stats.get_avg_process_latency() + last_invocation = self.stats.get_last_invocation() + + status.numReceived = int(total_received) if sys.version_info.major >= 3 else long(total_received) + status.numSuccessfullyProcessed = int(total_processed_successfully) if sys.version_info.major >= 3 else long(total_processed_successfully) + status.numUserExceptions = int(total_user_exceptions) if sys.version_info.major >= 3 else long(total_user_exceptions) + status.instanceId = self.instance_config.instance_id + for ex, tm in self.stats.latest_user_exception: + to_add = status.latestUserExceptions.add() + to_add.exceptionString = ex + to_add.msSinceEpoch = tm + status.numSystemExceptions = int(total_sys_exceptions) if sys.version_info.major >= 3 else long(total_sys_exceptions) + for ex, tm in self.stats.latest_sys_exception: + to_add = status.latestSystemExceptions.add() + to_add.exceptionString = ex + to_add.msSinceEpoch = tm + status.averageLatency = avg_process_latency_ms + status.lastInvocationTime = int(last_invocation) if sys.version_info.major >= 3 else long(last_invocation) + return status + + def join(self): + self.queue.put(InternalQuitMessage(True), True) + self.execution_thread.join() + self.close() + + def close(self): + Log.info("Closing python instance...") + if self.producer: + self.producer.close() + + if self.consumers: + for consumer in self.consumers.values(): + try: + consumer.close() + except: + pass + + if self.pulsar_client: + self.pulsar_client.close() diff --git a/conf/instances/python-instance/python_instance_main.py b/conf/instances/python-instance/python_instance_main.py new file mode 100644 index 0000000000000..627013489ab70 --- /dev/null +++ b/conf/instances/python-instance/python_instance_main.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""python_instance_main.py: The main for the Python Instance +""" +import argparse +import logging +import os +import sys +import signal +import time +import zipfile +import json +import inspect +import threading + +import pulsar + +import Function_pb2 +import log +import server +import python_instance +import util +# import prometheus_client +import prometheus_client_fix + +from google.protobuf import json_format +from bookkeeper.kv.client import Client + +to_run = True +Log = log.Log + +def atexit_function(signo, _frame): + global to_run + Log.info("Interrupted by %d, shutting down" % signo) + to_run = False + +def main(): + # Setup signal handlers + signal.signal(signal.SIGTERM, atexit_function) + signal.signal(signal.SIGHUP, atexit_function) + signal.signal(signal.SIGINT, atexit_function) + + parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance') + parser.add_argument('--function_details', required=True, help='Function Details Json String') + parser.add_argument('--py', required=True, help='Full Path of Function Code File') + parser.add_argument('--instance_id', required=True, help='Instance Id') + parser.add_argument('--function_id', required=True, help='Function Id') + parser.add_argument('--function_version', required=True, help='Function Version') + parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') + parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') + parser.add_argument('--client_auth_params', required=False, help='Client authentication params') + parser.add_argument('--use_tls', required=False, help='Use tls') + parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') + parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') + parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') + parser.add_argument('--port', required=True, help='Instance Port', type=int) + parser.add_argument('--metrics_port', required=True, help="Port metrics will be exposed on", type=int) + parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') + parser.add_argument('--logging_directory', required=True, help='Logging Directory') + parser.add_argument('--logging_file', required=True, help='Log file name') + parser.add_argument('--logging_config_file', required=True, help='Config file for logging') + parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) + parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider') + parser.add_argument('--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider') + parser.add_argument('--install_usercode_dependencies', required=False, help='For packaged python like wheel files, do we need to install all dependencies', type=bool) + parser.add_argument('--dependency_repository', required=False, help='For packaged python like wheel files, which repository to pull the dependencies from') + parser.add_argument('--extra_dependency_repository', required=False, help='For packaged python like wheel files, any extra repository to pull the dependencies from') + parser.add_argument('--state_storage_serviceurl', required=False, help='Managed State Storage Service Url') + parser.add_argument('--cluster_name', required=True, help='The name of the cluster this instance is running on') + + args = parser.parse_args() + function_details = Function_pb2.FunctionDetails() + args.function_details = str(args.function_details) + if args.function_details[0] == '\'': + args.function_details = args.function_details[1:] + if args.function_details[-1] == '\'': + args.function_details = args.function_details[:-1] + json_format.Parse(args.function_details, function_details) + + if os.path.splitext(str(args.py))[1] == '.whl': + if args.install_usercode_dependencies: + cmd = "pip install -t %s" % os.path.dirname(str(args.py)) + if args.dependency_repository: + cmd = cmd + " -i %s" % str(args.dependency_repository) + if args.extra_dependency_repository: + cmd = cmd + " --extra-index-url %s" % str(args.extra_dependency_repository) + cmd = cmd + " %s" % str(args.py) + retval = os.system(cmd) + if retval != 0: + print("Could not install user depedencies") + sys.exit(1) + else: + zpfile = zipfile.ZipFile(str(args.py), 'r') + zpfile.extractall(os.path.dirname(str(args.py))) + sys.path.insert(0, os.path.dirname(str(args.py))) + elif os.path.splitext(str(args.py))[1] == '.zip': + # Assumig zip file with format func.zip + # extract to folder function + # internal dir format + # "func/src" + # "func/requirements.txt" + # "func/deps" + # run pip install to target folder deps folder + zpfile = zipfile.ZipFile(str(args.py), 'r') + zpfile.extractall(os.path.dirname(str(args.py))) + basename = os.path.splitext(str(args.py))[0] + + deps_dir = os.path.join(os.path.dirname(str(args.py)), basename, "deps") + + if os.path.isdir(deps_dir) and os.listdir(deps_dir): + # get all wheel files from deps directory + wheel_file_list = [os.path.join(deps_dir, f) for f in os.listdir(deps_dir) if os.path.isfile(os.path.join(deps_dir, f)) and os.path.splitext(f)[1] =='.whl'] + cmd = "pip install -t %s --no-index --find-links %s %s" % (os.path.dirname(str(args.py)), deps_dir, " ".join(wheel_file_list)) + Log.debug("Install python dependencies via cmd: %s" % cmd) + retval = os.system(cmd) + if retval != 0: + print("Could not install user depedencies specified by the zip file") + sys.exit(1) + # add python user src directory to path + sys.path.insert(0, os.path.join(os.path.dirname(str(args.py)), basename, "src")) + + log_file = os.path.join(args.logging_directory, + util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), + "%s-%s.log" % (args.logging_file, args.instance_id)) + log.init_logger(logging.INFO, log_file, args.logging_config_file) + + Log.info("Starting Python instance with %s" % str(args)) + + authentication = None + use_tls = False + tls_allow_insecure_connection = False + tls_trust_cert_path = None + hostname_verification_enabled = False + if args.client_auth_plugin and args.client_auth_params: + authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) + if args.use_tls == "true": + use_tls = True + if args.tls_allow_insecure_connection == "true": + tls_allow_insecure_connection = True + if args.tls_trust_cert_path: + tls_trust_cert_path = args.tls_trust_cert_path + if args.hostname_verification_enabled == "true": + hostname_verification_enabled = True + pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication=authentication, operation_timeout_seconds=30, + io_threads=1, message_listener_threads=1, concurrent_lookup_requests=50000, + log_conf_file_path=None, use_tls=use_tls, tls_trust_certs_file_path=tls_trust_cert_path, + tls_allow_insecure_connection=tls_allow_insecure_connection, + tls_validate_hostname=hostname_verification_enabled) + + state_storage_serviceurl = None + if args.state_storage_serviceurl is not None: + state_storage_serviceurl = str(args.state_storage_serviceurl) + + secrets_provider = None + if args.secrets_provider is not None: + secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider)) + else: + secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider") + secrets_provider = secrets_provider() + secrets_provider_config = None + if args.secrets_provider_config is not None: + args.secrets_provider_config = str(args.secrets_provider_config) + if args.secrets_provider_config[0] == '\'': + args.secrets_provider_config = args.secrets_provider_config[1:] + if args.secrets_provider_config[-1] == '\'': + args.secrets_provider_config = args.secrets_provider_config[:-1] + secrets_provider_config = json.loads(str(args.secrets_provider_config)) + secrets_provider.init(secrets_provider_config) + + pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), + str(args.function_version), function_details, + int(args.max_buffered_tuples), + int(args.expected_healthcheck_interval), + str(args.py), + pulsar_client, + secrets_provider, + args.cluster_name, + state_storage_serviceurl) + pyinstance.run() + server_instance = server.serve(args.port, pyinstance) + + # Cannot use latest version of prometheus client because of thread leak + # prometheus_client.start_http_server(args.metrics_port) + # Use patched version of prometheus + # Contains fix from https://github.com/prometheus/client_python/pull/356 + # This can be removed one the fix in is a official prometheus client release + prometheus_client_fix.start_http_server(args.metrics_port) + + global to_run + while to_run: + time.sleep(1) + + pyinstance.join() + # make sure to close all non-daemon threads before this! + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/conf/instances/python-instance/secretsprovider.py b/conf/instances/python-instance/secretsprovider.py new file mode 100644 index 0000000000000..db8e68c094f66 --- /dev/null +++ b/conf/instances/python-instance/secretsprovider.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""secretsprovider.py: Interfaces and definitions for Secret Providers +""" +from abc import abstractmethod +import os + +class SecretsProvider: + """Interface for providing secrets information runtime""" + @abstractmethod + def init(self, config): + """Do any kind of initialization""" + pass + + @abstractmethod + def provide_secret(self, secret_name, path_to_secret): + """Fetches the secret located at the path""" + pass + + +"""A simple implementation that represents storing secrets in clear text """ +class ClearTextSecretsProvider(SecretsProvider): + def __init__(self): + pass + + def init(self, config): + pass + + def provide_secret(self, secret_name, path_to_secret): + return path_to_secret + +"""Implementation that fetches secrets from environment variables""" +class EnvironmentBasedSecretsProvider(SecretsProvider): + def __init__(self): + pass + + def init(self, config): + pass + + def provide_secret(self, secret_name, path_to_secret): + return os.environ.get(secret_name) diff --git a/conf/instances/python-instance/server.py b/conf/instances/python-instance/server.py new file mode 100644 index 0000000000000..58d43d204d4b1 --- /dev/null +++ b/conf/instances/python-instance/server.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""python_instance.py: Python Instance for running python functions +""" +from concurrent import futures +from log import Log +import grpc + +import InstanceCommunication_pb2_grpc + +class InstanceCommunicationServicer(InstanceCommunication_pb2_grpc.InstanceControlServicer): + """Provides methods that implement functionality of route guide server.""" + + def __init__(self, pyinstance): + self.pyinstance = pyinstance + + def GetFunctionStatus(self, request, context): + Log.debug("Came in GetFunctionStatus") + return self.pyinstance.get_function_status() + + def GetAndResetMetrics(self, request, context): + Log.debug("Came in GetAndResetMetrics") + return self.pyinstance.get_and_reset_metrics() + + def ResetMetrics(self, request, context): + Log.debug("Came in ResetMetrics") + self.pyinstance.reset_metrics() + return request + + def GetMetrics(self, request, context): + Log.debug("Came in GetMetrics") + return self.pyinstance.get_metrics() + + def HealthCheck(self, request, context): + return self.pyinstance.health_check() + + +def serve(port, pyinstance): + server_instance = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + InstanceCommunication_pb2_grpc.add_InstanceControlServicer_to_server( + InstanceCommunicationServicer(pyinstance), server_instance) + server_instance.add_insecure_port('[::]:%d' % port) + Log.info("Serving InstanceCommunication on port %d" % int(port)) + server_instance.start() + return server_instance diff --git a/conf/instances/python-instance/state_context.py b/conf/instances/python-instance/state_context.py new file mode 100644 index 0000000000000..87dcb5369b5c4 --- /dev/null +++ b/conf/instances/python-instance/state_context.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""state_context.py: state context for accessing managed state +""" +from abc import abstractmethod +from bookkeeper import admin, kv +from bookkeeper.common.exceptions import NamespaceNotFoundError, StreamNotFoundError, KeyNotFoundError +from bookkeeper.proto import stream_pb2 +from bookkeeper.proto.stream_pb2 import HASH +from bookkeeper.proto.stream_pb2 import TABLE +from bookkeeper.types import StorageClientSettings + + +def new_bk_table_conf(num_ranges): + """Create a table configuration with the specified `num_ranges`""" + return stream_pb2.StreamConfiguration( + key_type=HASH, + min_num_ranges=num_ranges, + initial_num_ranges=num_ranges, + split_policy=stream_pb2.SplitPolicy( + type=stream_pb2.SplitPolicyType.values()[0], + fixed_range_policy=stream_pb2.FixedRangeSplitPolicy( + num_ranges=2 + ) + ), + rolling_policy=stream_pb2.SegmentRollingPolicy( + size_policy=stream_pb2.SizeBasedSegmentRollingPolicy( + max_segment_size=128 * 1024 * 1024 + ) + ), + retention_policy=stream_pb2.RetentionPolicy( + time_policy=stream_pb2.TimeBasedRetentionPolicy( + retention_minutes=-1 + ) + ), + storage_type=TABLE + ) + + +def create_state_context(state_storage_serviceurl, table_ns, table_name): + """Create the state context based on state storage serviceurl""" + if state_storage_serviceurl is None: + return NullStateContext() + else: + return BKManagedStateContext(state_storage_serviceurl, table_ns, table_name) + + +class StateContext(object): + """Interface defining operations on managed state""" + + @abstractmethod + def incr(self, key, amount): + pass + + @abstractmethod + def put(self, key, value): + pass + + @abstractmethod + def get_value(self, key): + pass + + @abstractmethod + def get_amount(self, key): + pass + + +class NullStateContext(StateContext): + """A state context that does nothing""" + + def incr(self, key, amount): + return + + def put(self, key, value): + return + + def get_value(self, key): + return None + + def get_amount(self, key): + return None + + +class BKManagedStateContext(StateContext): + """A state context that access bookkeeper managed state""" + + def __init__(self, state_storage_serviceurl, table_ns, table_name): + client_settings = StorageClientSettings( + service_uri=state_storage_serviceurl) + admin_client = admin.client.Client( + storage_client_settings=client_settings) + # create namespace and table if needed + ns = admin_client.namespace(table_ns) + try: + ns.get(stream_name=table_name) + except NamespaceNotFoundError: + admin_client.namespaces().create(namespace=table_ns) + # TODO: make number of table ranges configurable + table_conf = new_bk_table_conf(1) + ns.create( + stream_name=table_name, + stream_config=table_conf) + except StreamNotFoundError: + # TODO: make number of table ranges configurable + table_conf = new_bk_table_conf(1) + ns.create( + stream_name=table_name, + stream_config=table_conf) + self.__client__ = kv.Client(storage_client_settings=client_settings, + namespace=table_ns) + self.__table__ = self.__client__.table(table_name=table_name) + + def incr(self, key, amount): + return self.__table__.incr_str(key, amount) + + def get_amount(self, key): + try: + kv = self.__table__.get_str(key) + if kv is not None: + return kv.number_value + else: + return None + except KeyNotFoundError: + return None + + def get_value(self, key): + try: + kv = self.__table__.get_str(key) + if kv is not None: + return kv.value + else: + return None + except KeyNotFoundError: + return None + + def put(self, key, value): + return self.__table__.put_str(key, value) + + def delete_key(self, key): + return self.__table__.delete_str(key) diff --git a/conf/instances/python-instance/tests/test_python_instance.py b/conf/instances/python-instance/tests/test_python_instance.py new file mode 100644 index 0000000000000..bbd6ca1fe42f1 --- /dev/null +++ b/conf/instances/python-instance/tests/test_python_instance.py @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# DEPENDENCIES: unittest2,mock + +from mock import Mock +import sys +sys.modules['prometheus_client'] = Mock() + +from contextimpl import ContextImpl +from python_instance import InstanceConfig +from pulsar import Message + +import Function_pb2 +import log +import os +import unittest + +class TestContextImpl(unittest.TestCase): + + def Any(cls): + class Any(cls): + def __eq__(self, other): + return True + return Any() + + def setUp(self): + log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini") + + def test_context_publish(self): + instance_id = 'test_instance_id' + function_id = 'test_function_id' + function_version = 'test_function_version' + function_details = Function_pb2.FunctionDetails() + max_buffered_tuples = 100; + instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) + logger = log.Log + pulsar_client = Mock() + producer = Mock() + producer.send_async = Mock(return_value=None) + pulsar_client.create_producer = Mock(return_value=producer) + user_code=__file__ + consumers = None + context_impl = ContextImpl(instance_config, logger, pulsar_client, user_code, consumers, None, None, None, None) + + msg = Message() + msg.message_id = Mock(return_value="test_message_id") + msg.partition_key = Mock(return_value="test_key") + context_impl.set_current_message_context(msg, "test_topic_name") + + context_impl.publish("test_topic_name", "test_message") + + args, kwargs = producer.send_async.call_args + self.assertEqual(args[0].decode("utf-8"), "test_message") + self.assertEqual(args[1].args[1], "test_topic_name") + self.assertEqual(args[1].args[2], "test_message_id") + + def test_context_ack_partitionedtopic(self): + instance_id = 'test_instance_id' + function_id = 'test_function_id' + function_version = 'test_function_version' + function_details = Function_pb2.FunctionDetails() + max_buffered_tuples = 100; + instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples) + logger = log.Log + pulsar_client = Mock() + user_code=__file__ + consumer = Mock() + consumer.acknowledge = Mock(return_value=None) + consumers = {"mytopic" : consumer} + context_impl = ContextImpl(instance_config, logger, pulsar_client, user_code, consumers, None, None, None, None) + context_impl.ack("test_message_id", "mytopic-partition-3") + + args, kwargs = consumer.acknowledge.call_args + self.assertEqual(args[0], "test_message_id") \ No newline at end of file diff --git a/conf/instances/python-instance/tests/test_secretsprovider.py b/conf/instances/python-instance/tests/test_secretsprovider.py new file mode 100644 index 0000000000000..5d725ee8c06e4 --- /dev/null +++ b/conf/instances/python-instance/tests/test_secretsprovider.py @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# DEPENDENCIES: unittest2,mock + +from secretsprovider import ClearTextSecretsProvider +from secretsprovider import EnvironmentBasedSecretsProvider + +import log +import os +import unittest + +class TestContextImpl(unittest.TestCase): + + def setUp(self): + log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini") + + def test_cleartext_secretsprovider(self): + provider = ClearTextSecretsProvider() + secret = provider.provide_secret("secretName", "secretPath") + self.assertEqual(secret, "secretPath") + secret = provider.provide_secret("secretName", "") + self.assertEqual(secret, "") + secret = provider.provide_secret("secretName", None) + self.assertEqual(secret, None) + + def test_environment_secretsprovider(self): + provider = EnvironmentBasedSecretsProvider() + secret = provider.provide_secret("secretName", "secretPath") + self.assertEqual(secret, None) + os.environ["secretName"] = "secretValue" + secret = provider.provide_secret("secretName", "") + self.assertEqual(secret, "secretValue") + secret = provider.provide_secret("secretName", None) + self.assertEqual(secret, "secretValue") + secret = provider.provide_secret("secretName", "somethingelse") + self.assertEqual(secret, "secretValue") \ No newline at end of file diff --git a/conf/instances/python-instance/util.py b/conf/instances/python-instance/util.py new file mode 100644 index 0000000000000..390aed1b77497 --- /dev/null +++ b/conf/instances/python-instance/util.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# -*- encoding: utf-8 -*- + +"""util.py: Some misc utility functions +""" +import os +import inspect +import sys +import importlib +from threading import Timer + +import log + +Log = log.Log +PULSAR_API_ROOT = 'pulsar' +PULSAR_FUNCTIONS_API_ROOT = 'functions' + +def import_class(from_path, full_class_name): + from_path = str(from_path) + full_class_name = str(full_class_name) + try: + return import_class_from_path(from_path, full_class_name) + except Exception as e: + our_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + api_dir = os.path.join(our_dir, PULSAR_API_ROOT, PULSAR_FUNCTIONS_API_ROOT) + try: + return import_class_from_path(api_dir, full_class_name) + except Exception as e: + Log.info("Failed to import class %s from path %s" % (full_class_name, from_path)) + Log.info(e, exc_info=True) + return None + +def import_class_from_path(from_path, full_class_name): + Log.debug('Trying to import %s from path %s' % (full_class_name, from_path)) + split = full_class_name.split('.') + classname_path = '.'.join(split[:-1]) + class_name = full_class_name.split('.')[-1] + if from_path not in sys.path: + Log.debug("Add a new dependency to the path: %s" % from_path) + sys.path.insert(0, from_path) + if not classname_path: + mod = importlib.import_module(class_name) + return mod + else: + mod = importlib.import_module(classname_path) + retval = getattr(mod, class_name) + return retval + +def getFullyQualifiedFunctionName(tenant, namespace, name): + return "%s/%s/%s" % (tenant, namespace, name) + +def getFullyQualifiedInstanceId(tenant, namespace, name, instance_id): + return "%s/%s/%s:%s" % (tenant, namespace, name, instance_id) + +def get_properties(fullyQualifiedName, instanceId): + return {"application": "pulsar-function", "id": str(fullyQualifiedName), "instance_id": str(instanceId)} + +class FixedTimer(): + + def __init__(self, t, hFunction, name="timer-thread"): + self.t = t + self.hFunction = hFunction + self.thread = Timer(self.t, self.handle_function) + self.thread.setName(name) + self.thread.setDaemon(True) + + def handle_function(self): + self.hFunction() + self.thread = Timer(self.t, self.handle_function) + self.thread.start() + + def start(self): + self.thread.start() + + def cancel(self): + self.thread.cancel() \ No newline at end of file diff --git a/conf/my-secret.key b/conf/my-secret.key new file mode 100644 index 0000000000000..dcfbf326c05a9 --- /dev/null +++ b/conf/my-secret.key @@ -0,0 +1 @@ +C4/Q5QrwdJDNmMN9QzAfgR01Q6lod9iWu462zYqPoOw= \ No newline at end of file diff --git a/conf/pulsar-auth-basic.conf b/conf/pulsar-auth-basic.conf new file mode 100644 index 0000000000000..3d7d6c55bac4b --- /dev/null +++ b/conf/pulsar-auth-basic.conf @@ -0,0 +1 @@ +lan:$apr1$zGv70FBE$e53WUIAqNsGu5uVELGWRY. diff --git a/conf/rocksdb.conf b/conf/rocksdb.conf new file mode 100644 index 0000000000000..1affbc0ce3f28 --- /dev/null +++ b/conf/rocksdb.conf @@ -0,0 +1,8 @@ +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 +[CFOptions "default"] diff --git a/conf/standalone.conf b/conf/standalone.conf index b1b3a068e35c2..7637f78b806cd 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -17,6 +17,12 @@ # under the License. # + +#messagingProtocols=mqtt +protocolHandlerDirectory=conf/protocols +mqttListeners=mqtt://0.0.0.0:1883 +hostName=0.0.0.0 + ### --- General broker settings --- ### # The metadata store URL @@ -37,6 +43,7 @@ brokerServicePort=6650 # Port to use to server HTTP request webServicePort=8080 +# webServicePortTls=8081 # Hostname or IP address the service binds on, default is 0.0.0.0. bindAddress=0.0.0.0 @@ -510,7 +517,8 @@ authenticateOriginalAuthData=false authenticationEnabled=false # Authentication provider name list, which is comma separated list of class names -authenticationProviders= +#authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderBasic +authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.AuthenticationProviderBasic # Enforce authorization authorizationEnabled=false @@ -525,18 +533,21 @@ authorizationAllowWildcardsMatching=false # Role names that are treated as "super-user", meaning they will be able to do all admin # operations and publish/consume from all topics -superUserRoles= +superUserRoles=admin # Authentication settings of the broker itself. Used when the broker connects to other brokers, # either in same or other clusters -brokerClientAuthenticationPlugin= -brokerClientAuthenticationParameters= +brokerClientTlsEnabled=false +# brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationBasic +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationToken +#brokerClientAuthenticationParameters={"token":"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.WkgQ1xTmRikzq-qMYlLD1wWdA0Ur0-1cO6N3mhKt8yI"} +brokerClientAuthenticationParameters={"token":"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.WkgQ1xTmRikzq-qMYlLD1wWdA0Ur0-1cO6N3mhKt8yI","userId":"lan","password":"lan"} # Supported Athenz provider domain names(comma separated) for authentication athenzDomainNames= # When this parameter is not empty, unauthenticated users perform as anonymousUserRole -anonymousUserRole= +anonymousUserRole=lan ## Configure the datasource of basic authenticate, supports the file and Base64 format. # file: @@ -552,7 +563,7 @@ basicAuthConf= # The key can be specified like: # tokenSecretKey=data:;base64,xxxxxxxxx # tokenSecretKey=file:///my/secret.key ( Note: key file must be DER-encoded ) -tokenSecretKey= +tokenSecretKey=/home/lan/repo/git/pulsar/conf/my-secret.key ## Asymmetric public/private key pair # Configure the public key to be used to validate auth tokens @@ -1105,7 +1116,7 @@ gcWaitTime=300000 allowAutoTopicCreation=true # The type of topic that is allowed to be automatically created.(partitioned/non-partitioned) -allowAutoTopicCreationType=non-partitioned +allowAutoTopicCreationType=partitioned # Enable subscription auto creation if new consumer connected (disable auto creation with value false) allowAutoSubscriptionCreation=true @@ -1172,10 +1183,11 @@ transactionPendingAckBatchedWriteMaxDelayInMillis=1 ### --- Packages management service configuration variables (begin) --- ### # Enable the packages management service or not -enablePackagesManagement=false +enablePackagesManagement=true # The packages management service storage service provide -packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider +#packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider +packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.filesystem.FileSystemPackagesStorageProvider # When the packages storage provider is bookkeeper, you can use this configuration to # control the number of replicas for storing the package diff --git a/devmetadata/CURRENT b/devmetadata/CURRENT new file mode 100644 index 0000000000000..cacca7574c034 --- /dev/null +++ b/devmetadata/CURRENT @@ -0,0 +1 @@ +MANIFEST-000004 diff --git a/devmetadata/IDENTITY b/devmetadata/IDENTITY new file mode 100644 index 0000000000000..5e3ef39607278 --- /dev/null +++ b/devmetadata/IDENTITY @@ -0,0 +1 @@ +c354e1c3-6078-4c1c-9de6-c8da942562a5 \ No newline at end of file diff --git a/devmetadata/LOCK b/devmetadata/LOCK new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/devmetadata/LOG b/devmetadata/LOG new file mode 100644 index 0000000000000..cbbe86003646b --- /dev/null +++ b/devmetadata/LOG @@ -0,0 +1,318 @@ +2022/08/12-15:11:06.943597 139707605608192 RocksDB version: 6.29.4 +2022/08/12-15:11:06.944140 139707605608192 Compile date 2022-03-23 13:34:32 +2022/08/12-15:11:06.944164 139707605608192 DB SUMMARY +2022/08/12-15:11:06.944169 139707605608192 DB Session ID: 2XGYZ12Y0LMR4D2WIMAI +2022/08/12-15:11:06.944218 139707605608192 SST files in /home/lan/repo/git/pulsar/devmetadata dir, Total Num: 0, files: +2022/08/12-15:11:06.944224 139707605608192 Write Ahead Log file in /home/lan/repo/git/pulsar/devmetadata: +2022/08/12-15:11:06.944229 139707605608192 Options.error_if_exists: 0 +2022/08/12-15:11:06.944232 139707605608192 Options.create_if_missing: 1 +2022/08/12-15:11:06.944235 139707605608192 Options.paranoid_checks: 1 +2022/08/12-15:11:06.944237 139707605608192 Options.flush_verify_memtable_count: 1 +2022/08/12-15:11:06.944240 139707605608192 Options.track_and_verify_wals_in_manifest: 0 +2022/08/12-15:11:06.944243 139707605608192 Options.env: 0x7f0df15f1ac0 +2022/08/12-15:11:06.944246 139707605608192 Options.fs: PosixFileSystem +2022/08/12-15:11:06.944249 139707605608192 Options.info_log: 0x7f10307386c0 +2022/08/12-15:11:06.944252 139707605608192 Options.max_file_opening_threads: 16 +2022/08/12-15:11:06.944255 139707605608192 Options.statistics: (nil) +2022/08/12-15:11:06.944257 139707605608192 Options.use_fsync: 0 +2022/08/12-15:11:06.944259 139707605608192 Options.max_log_file_size: 0 +2022/08/12-15:11:06.944262 139707605608192 Options.max_manifest_file_size: 1073741824 +2022/08/12-15:11:06.944264 139707605608192 Options.log_file_time_to_roll: 86400 +2022/08/12-15:11:06.944267 139707605608192 Options.keep_log_file_num: 30 +2022/08/12-15:11:06.944269 139707605608192 Options.recycle_log_file_num: 0 +2022/08/12-15:11:06.944272 139707605608192 Options.allow_fallocate: 1 +2022/08/12-15:11:06.944274 139707605608192 Options.allow_mmap_reads: 0 +2022/08/12-15:11:06.944276 139707605608192 Options.allow_mmap_writes: 0 +2022/08/12-15:11:06.944278 139707605608192 Options.use_direct_reads: 0 +2022/08/12-15:11:06.944281 139707605608192 Options.use_direct_io_for_flush_and_compaction: 0 +2022/08/12-15:11:06.944283 139707605608192 Options.create_missing_column_families: 0 +2022/08/12-15:11:06.944286 139707605608192 Options.db_log_dir: +2022/08/12-15:11:06.944288 139707605608192 Options.wal_dir: +2022/08/12-15:11:06.944291 139707605608192 Options.table_cache_numshardbits: 6 +2022/08/12-15:11:06.944293 139707605608192 Options.WAL_ttl_seconds: 0 +2022/08/12-15:11:06.944295 139707605608192 Options.WAL_size_limit_MB: 0 +2022/08/12-15:11:06.944298 139707605608192 Options.max_write_batch_group_size_bytes: 1048576 +2022/08/12-15:11:06.944300 139707605608192 Options.manifest_preallocation_size: 4194304 +2022/08/12-15:11:06.944302 139707605608192 Options.is_fd_close_on_exec: 1 +2022/08/12-15:11:06.944305 139707605608192 Options.advise_random_on_open: 1 +2022/08/12-15:11:06.944310 139707605608192 Options.experimental_mempurge_threshold: 0.000000 +2022/08/12-15:11:06.944313 139707605608192 Options.db_write_buffer_size: 0 +2022/08/12-15:11:06.944315 139707605608192 Options.write_buffer_manager: 0x7f1030d52f70 +2022/08/12-15:11:06.944317 139707605608192 Options.access_hint_on_compaction_start: 1 +2022/08/12-15:11:06.944320 139707605608192 Options.new_table_reader_for_compaction_inputs: 0 +2022/08/12-15:11:06.944322 139707605608192 Options.random_access_max_buffer_size: 1048576 +2022/08/12-15:11:06.944324 139707605608192 Options.use_adaptive_mutex: 0 +2022/08/12-15:11:06.944354 139707605608192 Options.rate_limiter: (nil) +2022/08/12-15:11:06.944425 139707605608192 Options.sst_file_manager.rate_bytes_per_sec: 0 +2022/08/12-15:11:06.944429 139707605608192 Options.wal_recovery_mode: 2 +2022/08/12-15:11:06.944432 139707605608192 Options.enable_thread_tracking: 0 +2022/08/12-15:11:06.944434 139707605608192 Options.enable_pipelined_write: 0 +2022/08/12-15:11:06.944436 139707605608192 Options.unordered_write: 0 +2022/08/12-15:11:06.944438 139707605608192 Options.allow_concurrent_memtable_write: 1 +2022/08/12-15:11:06.944441 139707605608192 Options.enable_write_thread_adaptive_yield: 1 +2022/08/12-15:11:06.944443 139707605608192 Options.write_thread_max_yield_usec: 100 +2022/08/12-15:11:06.944445 139707605608192 Options.write_thread_slow_yield_usec: 3 +2022/08/12-15:11:06.944448 139707605608192 Options.row_cache: None +2022/08/12-15:11:06.944450 139707605608192 Options.wal_filter: None +2022/08/12-15:11:06.944452 139707605608192 Options.avoid_flush_during_recovery: 0 +2022/08/12-15:11:06.944455 139707605608192 Options.allow_ingest_behind: 0 +2022/08/12-15:11:06.944457 139707605608192 Options.preserve_deletes: 0 +2022/08/12-15:11:06.944459 139707605608192 Options.two_write_queues: 0 +2022/08/12-15:11:06.944461 139707605608192 Options.manual_wal_flush: 0 +2022/08/12-15:11:06.944464 139707605608192 Options.atomic_flush: 0 +2022/08/12-15:11:06.944548 139707605608192 Options.avoid_unnecessary_blocking_io: 0 +2022/08/12-15:11:06.944553 139707605608192 Options.persist_stats_to_disk: 0 +2022/08/12-15:11:06.944555 139707605608192 Options.write_dbid_to_manifest: 0 +2022/08/12-15:11:06.944557 139707605608192 Options.log_readahead_size: 0 +2022/08/12-15:11:06.944559 139707605608192 Options.file_checksum_gen_factory: Unknown +2022/08/12-15:11:06.944563 139707605608192 Options.best_efforts_recovery: 0 +2022/08/12-15:11:06.944565 139707605608192 Options.max_bgerror_resume_count: 2147483647 +2022/08/12-15:11:06.944568 139707605608192 Options.bgerror_resume_retry_interval: 1000000 +2022/08/12-15:11:06.944570 139707605608192 Options.allow_data_in_errors: 0 +2022/08/12-15:11:06.944573 139707605608192 Options.db_host_id: __hostname__ +2022/08/12-15:11:06.944578 139707605608192 Options.max_background_jobs: 2 +2022/08/12-15:11:06.944580 139707605608192 Options.max_background_compactions: -1 +2022/08/12-15:11:06.944583 139707605608192 Options.max_subcompactions: 1 +2022/08/12-15:11:06.944585 139707605608192 Options.avoid_flush_during_shutdown: 0 +2022/08/12-15:11:06.944587 139707605608192 Options.writable_file_max_buffer_size: 1048576 +2022/08/12-15:11:06.944590 139707605608192 Options.delayed_write_rate : 16777216 +2022/08/12-15:11:06.944592 139707605608192 Options.max_total_wal_size: 0 +2022/08/12-15:11:06.944595 139707605608192 Options.delete_obsolete_files_period_micros: 21600000000 +2022/08/12-15:11:06.944598 139707605608192 Options.stats_dump_period_sec: 600 +2022/08/12-15:11:06.944600 139707605608192 Options.stats_persist_period_sec: 600 +2022/08/12-15:11:06.944603 139707605608192 Options.stats_history_buffer_size: 1048576 +2022/08/12-15:11:06.944605 139707605608192 Options.max_open_files: -1 +2022/08/12-15:11:06.944608 139707605608192 Options.bytes_per_sync: 0 +2022/08/12-15:11:06.944610 139707605608192 Options.wal_bytes_per_sync: 0 +2022/08/12-15:11:06.944612 139707605608192 Options.strict_bytes_per_sync: 0 +2022/08/12-15:11:06.944614 139707605608192 Options.compaction_readahead_size: 0 +2022/08/12-15:11:06.944641 139707605608192 Options.max_background_flushes: -1 +2022/08/12-15:11:06.944644 139707605608192 Compression algorithms supported: +2022/08/12-15:11:06.944647 139707605608192 kZSTDNotFinalCompression supported: 1 +2022/08/12-15:11:06.945052 139707605608192 kZSTD supported: 1 +2022/08/12-15:11:06.945060 139707605608192 kXpressCompression supported: 0 +2022/08/12-15:11:06.945063 139707605608192 kLZ4HCCompression supported: 1 +2022/08/12-15:11:06.945066 139707605608192 kLZ4Compression supported: 1 +2022/08/12-15:11:06.945069 139707605608192 kBZip2Compression supported: 1 +2022/08/12-15:11:06.945072 139707605608192 kZlibCompression supported: 1 +2022/08/12-15:11:06.945074 139707605608192 kSnappyCompression supported: 1 +2022/08/12-15:11:06.945086 139707605608192 Fast CRC32 supported: Not supported on x86 +2022/08/12-15:11:06.952857 139707605608192 [/db_impl/db_impl_open.cc:307] Creating manifest 1 +2022/08/12-15:11:06.971896 139707605608192 [/version_set.cc:4890] Recovering from manifest file: /home/lan/repo/git/pulsar/devmetadata/MANIFEST-000001 +2022/08/12-15:11:06.972198 139707605608192 [/column_family.cc:607] --------------- Options for column family [default]: +2022/08/12-15:11:06.972208 139707605608192 Options.comparator: leveldb.BytewiseComparator +2022/08/12-15:11:06.972211 139707605608192 Options.merge_operator: None +2022/08/12-15:11:06.972214 139707605608192 Options.compaction_filter: None +2022/08/12-15:11:06.972216 139707605608192 Options.compaction_filter_factory: None +2022/08/12-15:11:06.972219 139707605608192 Options.sst_partitioner_factory: None +2022/08/12-15:11:06.972222 139707605608192 Options.memtable_factory: SkipListFactory +2022/08/12-15:11:06.972225 139707605608192 Options.table_factory: BlockBasedTable +2022/08/12-15:11:06.972262 139707605608192 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f1030d4fb10) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 1 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + data_block_index_type: 0 + index_shortening: 1 + data_block_hash_table_util_ratio: 0.750000 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7f10300b6c80 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: (nil) + persistent_cache: (nil) + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 5 + enable_index_compression: 1 + block_align: 0 + max_auto_readahead_size: 262144 + prepopulate_block_cache: 0 +2022/08/12-15:11:06.972267 139707605608192 Options.write_buffer_size: 67108864 +2022/08/12-15:11:06.972270 139707605608192 Options.max_write_buffer_number: 2 +2022/08/12-15:11:06.972273 139707605608192 Options.compression: Snappy +2022/08/12-15:11:06.972276 139707605608192 Options.bottommost_compression: Disabled +2022/08/12-15:11:06.972278 139707605608192 Options.prefix_extractor: nullptr +2022/08/12-15:11:06.972281 139707605608192 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2022/08/12-15:11:06.972284 139707605608192 Options.num_levels: 7 +2022/08/12-15:11:06.972286 139707605608192 Options.min_write_buffer_number_to_merge: 1 +2022/08/12-15:11:06.972288 139707605608192 Options.max_write_buffer_number_to_maintain: 0 +2022/08/12-15:11:06.972291 139707605608192 Options.max_write_buffer_size_to_maintain: 134217728 +2022/08/12-15:11:06.972294 139707605608192 Options.bottommost_compression_opts.window_bits: -14 +2022/08/12-15:11:06.972503 139707605608192 Options.bottommost_compression_opts.level: 32767 +2022/08/12-15:11:06.972508 139707605608192 Options.bottommost_compression_opts.strategy: 0 +2022/08/12-15:11:06.972511 139707605608192 Options.bottommost_compression_opts.max_dict_bytes: 0 +2022/08/12-15:11:06.972513 139707605608192 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2022/08/12-15:11:06.972515 139707605608192 Options.bottommost_compression_opts.parallel_threads: 1 +2022/08/12-15:11:06.972518 139707605608192 Options.bottommost_compression_opts.enabled: false +2022/08/12-15:11:06.972520 139707605608192 Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 +2022/08/12-15:11:06.972523 139707605608192 Options.compression_opts.window_bits: -14 +2022/08/12-15:11:06.972526 139707605608192 Options.compression_opts.level: 32767 +2022/08/12-15:11:06.972529 139707605608192 Options.compression_opts.strategy: 0 +2022/08/12-15:11:06.972531 139707605608192 Options.compression_opts.max_dict_bytes: 0 +2022/08/12-15:11:06.972534 139707605608192 Options.compression_opts.zstd_max_train_bytes: 0 +2022/08/12-15:11:06.972536 139707605608192 Options.compression_opts.parallel_threads: 1 +2022/08/12-15:11:06.972539 139707605608192 Options.compression_opts.enabled: false +2022/08/12-15:11:06.972542 139707605608192 Options.compression_opts.max_dict_buffer_bytes: 0 +2022/08/12-15:11:06.972545 139707605608192 Options.level0_file_num_compaction_trigger: 4 +2022/08/12-15:11:06.972547 139707605608192 Options.level0_slowdown_writes_trigger: 20 +2022/08/12-15:11:06.972550 139707605608192 Options.level0_stop_writes_trigger: 36 +2022/08/12-15:11:06.972552 139707605608192 Options.target_file_size_base: 67108864 +2022/08/12-15:11:06.972555 139707605608192 Options.target_file_size_multiplier: 1 +2022/08/12-15:11:06.972558 139707605608192 Options.max_bytes_for_level_base: 268435456 +2022/08/12-15:11:06.972561 139707605608192 Options.level_compaction_dynamic_level_bytes: 0 +2022/08/12-15:11:06.972566 139707605608192 Options.max_bytes_for_level_multiplier: 10.000000 +2022/08/12-15:11:06.972569 139707605608192 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2022/08/12-15:11:06.972572 139707605608192 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2022/08/12-15:11:06.972575 139707605608192 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2022/08/12-15:11:06.972578 139707605608192 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2022/08/12-15:11:06.972580 139707605608192 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2022/08/12-15:11:06.972583 139707605608192 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2022/08/12-15:11:06.972586 139707605608192 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2022/08/12-15:11:06.972589 139707605608192 Options.max_sequential_skip_in_iterations: 8 +2022/08/12-15:11:06.972591 139707605608192 Options.max_compaction_bytes: 1677721600 +2022/08/12-15:11:06.972594 139707605608192 Options.arena_block_size: 1048576 +2022/08/12-15:11:06.972597 139707605608192 Options.soft_pending_compaction_bytes_limit: 68719476736 +2022/08/12-15:11:06.972600 139707605608192 Options.hard_pending_compaction_bytes_limit: 274877906944 +2022/08/12-15:11:06.972602 139707605608192 Options.rate_limit_delay_max_milliseconds: 100 +2022/08/12-15:11:06.972605 139707605608192 Options.disable_auto_compactions: 1 +2022/08/12-15:11:06.972608 139707605608192 Options.compaction_style: kCompactionStyleLevel +2022/08/12-15:11:06.972611 139707605608192 Options.compaction_pri: kMinOverlappingRatio +2022/08/12-15:11:06.972614 139707605608192 Options.compaction_options_universal.size_ratio: 1 +2022/08/12-15:11:06.972616 139707605608192 Options.compaction_options_universal.min_merge_width: 2 +2022/08/12-15:11:06.972811 139707605608192 Options.compaction_options_universal.max_merge_width: 4294967295 +2022/08/12-15:11:06.972816 139707605608192 Options.compaction_options_universal.max_size_amplification_percent: 200 +2022/08/12-15:11:06.972819 139707605608192 Options.compaction_options_universal.compression_size_percent: -1 +2022/08/12-15:11:06.972822 139707605608192 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2022/08/12-15:11:06.972825 139707605608192 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2022/08/12-15:11:06.972827 139707605608192 Options.compaction_options_fifo.allow_compaction: 0 +2022/08/12-15:11:06.972836 139707605608192 Options.table_properties_collectors: +2022/08/12-15:11:06.972839 139707605608192 Options.inplace_update_support: 0 +2022/08/12-15:11:06.972842 139707605608192 Options.inplace_update_num_locks: 10000 +2022/08/12-15:11:06.972846 139707605608192 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2022/08/12-15:11:06.972849 139707605608192 Options.memtable_whole_key_filtering: 0 +2022/08/12-15:11:06.972851 139707605608192 Options.memtable_huge_page_size: 0 +2022/08/12-15:11:06.972854 139707605608192 Options.bloom_locality: 0 +2022/08/12-15:11:06.972857 139707605608192 Options.max_successive_merges: 0 +2022/08/12-15:11:06.972860 139707605608192 Options.optimize_filters_for_hits: 0 +2022/08/12-15:11:06.972862 139707605608192 Options.paranoid_file_checks: 0 +2022/08/12-15:11:06.972865 139707605608192 Options.force_consistency_checks: 1 +2022/08/12-15:11:06.972867 139707605608192 Options.report_bg_io_stats: 0 +2022/08/12-15:11:06.972870 139707605608192 Options.ttl: 2592000 +2022/08/12-15:11:06.972873 139707605608192 Options.periodic_compaction_seconds: 0 +2022/08/12-15:11:06.972875 139707605608192 Options.enable_blob_files: false +2022/08/12-15:11:06.972878 139707605608192 Options.min_blob_size: 0 +2022/08/12-15:11:06.972881 139707605608192 Options.blob_file_size: 268435456 +2022/08/12-15:11:06.972884 139707605608192 Options.blob_compression_type: NoCompression +2022/08/12-15:11:06.972887 139707605608192 Options.enable_blob_garbage_collection: false +2022/08/12-15:11:06.972890 139707605608192 Options.blob_garbage_collection_age_cutoff: 0.250000 +2022/08/12-15:11:06.972894 139707605608192 Options.blob_garbage_collection_force_threshold: 1.000000 +2022/08/12-15:11:06.972897 139707605608192 Options.blob_compaction_readahead_size: 0 +2022/08/12-15:11:06.980999 139707605608192 [/version_set.cc:4938] Recovered from manifest file:/home/lan/repo/git/pulsar/devmetadata/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2022/08/12-15:11:06.981014 139707605608192 [/version_set.cc:4947] Column family [default] (ID 0), log number is 0 +2022/08/12-15:11:06.981158 139707605608192 [/version_set.cc:4410] Creating manifest 4 +2022/08/12-15:11:07.008218 139707605608192 [/db_impl/db_impl_open.cc:1799] SstFileManager instance 0x7f10309d90d0 +2022/08/12-15:11:07.008256 139707605608192 DB pointer 0x7f1030738bc0 +2022/08/12-15:11:07.008337 139707605608192 [WARN] [ilities/transactions/pessimistic_transaction_db.cc:245] Transaction write_policy is 0 +2022/08/12-15:11:07.008499 139696302655232 [/db_impl/db_impl.cc:1026] ------- DUMPING STATS ------- +2022/08/12-15:11:07.008524 139696302655232 [/db_impl/db_impl.cc:1027] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Blob file count: 0, total size: 0.0 GB + +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count +Block cache LRUCache@0x7f10300b6c80#2423292 capacity: 8.00 MB collections: 1 last_copies: 0 last_secs: 4.8e-05 secs_since: 0 +Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) + +** File Read Latency Histogram By Level [default] ** +2022/08/12-15:11:07.021308 139707605608192 [/db_impl/db_impl.cc:1130] SetOptions() on column family [default], inputs: +2022/08/12-15:11:07.021320 139707605608192 [/db_impl/db_impl.cc:1133] disable_auto_compactions: false +2022/08/12-15:11:07.021323 139707605608192 [/db_impl/db_impl.cc:1137] [default] SetOptions() succeeded +2022/08/12-15:11:07.021328 139707605608192 [tions/cf_options.cc:946] write_buffer_size: 67108864 +2022/08/12-15:11:07.021330 139707605608192 [tions/cf_options.cc:948] max_write_buffer_number: 2 +2022/08/12-15:11:07.021332 139707605608192 [tions/cf_options.cc:951] arena_block_size: 1048576 +2022/08/12-15:11:07.021334 139707605608192 [tions/cf_options.cc:953] memtable_prefix_bloom_ratio: 0.000000 +2022/08/12-15:11:07.021338 139707605608192 [tions/cf_options.cc:955] memtable_whole_key_filtering: 0 +2022/08/12-15:11:07.021340 139707605608192 [tions/cf_options.cc:958] memtable_huge_page_size: 0 +2022/08/12-15:11:07.021342 139707605608192 [tions/cf_options.cc:961] max_successive_merges: 0 +2022/08/12-15:11:07.021344 139707605608192 [tions/cf_options.cc:964] inplace_update_num_locks: 10000 +2022/08/12-15:11:07.021346 139707605608192 [tions/cf_options.cc:968] prefix_extractor: nullptr +2022/08/12-15:11:07.021348 139707605608192 [tions/cf_options.cc:970] disable_auto_compactions: 0 +2022/08/12-15:11:07.021350 139707605608192 [tions/cf_options.cc:972] soft_pending_compaction_bytes_limit: 68719476736 +2022/08/12-15:11:07.021352 139707605608192 [tions/cf_options.cc:974] hard_pending_compaction_bytes_limit: 274877906944 +2022/08/12-15:11:07.021354 139707605608192 [tions/cf_options.cc:976] level0_file_num_compaction_trigger: 4 +2022/08/12-15:11:07.021355 139707605608192 [tions/cf_options.cc:978] level0_slowdown_writes_trigger: 20 +2022/08/12-15:11:07.021357 139707605608192 [tions/cf_options.cc:980] level0_stop_writes_trigger: 36 +2022/08/12-15:11:07.021359 139707605608192 [tions/cf_options.cc:982] max_compaction_bytes: 1677721600 +2022/08/12-15:11:07.021361 139707605608192 [tions/cf_options.cc:984] target_file_size_base: 67108864 +2022/08/12-15:11:07.021363 139707605608192 [tions/cf_options.cc:986] target_file_size_multiplier: 1 +2022/08/12-15:11:07.021364 139707605608192 [tions/cf_options.cc:988] max_bytes_for_level_base: 268435456 +2022/08/12-15:11:07.021366 139707605608192 [tions/cf_options.cc:990] max_bytes_for_level_multiplier: 10.000000 +2022/08/12-15:11:07.021370 139707605608192 [tions/cf_options.cc:992] ttl: 2592000 +2022/08/12-15:11:07.021373 139707605608192 [tions/cf_options.cc:994] periodic_compaction_seconds: 0 +2022/08/12-15:11:07.021378 139707605608192 [tions/cf_options.cc:1008] max_bytes_for_level_multiplier_additional: 1, 1, 1, 1, 1, 1, 1 +2022/08/12-15:11:07.021380 139707605608192 [tions/cf_options.cc:1010] max_sequential_skip_in_iterations: 8 +2022/08/12-15:11:07.021382 139707605608192 [tions/cf_options.cc:1012] check_flush_compaction_key_order: 1 +2022/08/12-15:11:07.021383 139707605608192 [tions/cf_options.cc:1014] paranoid_file_checks: 0 +2022/08/12-15:11:07.021385 139707605608192 [tions/cf_options.cc:1016] report_bg_io_stats: 0 +2022/08/12-15:11:07.021387 139707605608192 [tions/cf_options.cc:1018] compression: 1 +2022/08/12-15:11:07.021389 139707605608192 [tions/cf_options.cc:1022] compaction_options_universal.size_ratio : 1 +2022/08/12-15:11:07.021391 139707605608192 [tions/cf_options.cc:1024] compaction_options_universal.min_merge_width : 2 +2022/08/12-15:11:07.021393 139707605608192 [tions/cf_options.cc:1026] compaction_options_universal.max_merge_width : -1 +2022/08/12-15:11:07.021395 139707605608192 [tions/cf_options.cc:1029] compaction_options_universal.max_size_amplification_percent : 200 +2022/08/12-15:11:07.021429 139707605608192 [tions/cf_options.cc:1032] compaction_options_universal.compression_size_percent : -1 +2022/08/12-15:11:07.021432 139707605608192 [tions/cf_options.cc:1034] compaction_options_universal.stop_style : 1 +2022/08/12-15:11:07.021434 139707605608192 [tions/cf_options.cc:1037] compaction_options_universal.allow_trivial_move : 0 +2022/08/12-15:11:07.021437 139707605608192 [tions/cf_options.cc:1039] compaction_options_universal.incremental : 0 +2022/08/12-15:11:07.021439 139707605608192 [tions/cf_options.cc:1043] compaction_options_fifo.max_table_files_size : 1073741824 +2022/08/12-15:11:07.021441 139707605608192 [tions/cf_options.cc:1045] compaction_options_fifo.allow_compaction : 0 +2022/08/12-15:11:07.021443 139707605608192 [tions/cf_options.cc:1049] enable_blob_files: false +2022/08/12-15:11:07.021446 139707605608192 [tions/cf_options.cc:1051] min_blob_size: 0 +2022/08/12-15:11:07.021448 139707605608192 [tions/cf_options.cc:1053] blob_file_size: 268435456 +2022/08/12-15:11:07.021451 139707605608192 [tions/cf_options.cc:1055] blob_compression_type: NoCompression +2022/08/12-15:11:07.021453 139707605608192 [tions/cf_options.cc:1057] enable_blob_garbage_collection: false +2022/08/12-15:11:07.021456 139707605608192 [tions/cf_options.cc:1059] blob_garbage_collection_age_cutoff: 0.250000 +2022/08/12-15:11:07.021459 139707605608192 [tions/cf_options.cc:1061] blob_garbage_collection_force_threshold: 1.000000 +2022/08/12-15:11:07.021461 139707605608192 [tions/cf_options.cc:1063] blob_compaction_readahead_size: 0 diff --git a/devmetadata/MANIFEST-000004 b/devmetadata/MANIFEST-000004 new file mode 100644 index 0000000000000..ae48d665368a1 Binary files /dev/null and b/devmetadata/MANIFEST-000004 differ diff --git a/devmetadata/OPTIONS-000007 b/devmetadata/OPTIONS-000007 new file mode 100644 index 0000000000000..0fc90b1e571ee --- /dev/null +++ b/devmetadata/OPTIONS-000007 @@ -0,0 +1,190 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.29.4 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + wal_bytes_per_sync=0 + bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + stats_dump_period_sec=600 + stats_persist_period_sec=600 + delete_obsolete_files_period_micros=21600000000 + max_total_wal_size=0 + strict_bytes_per_sync=false + delayed_write_rate=16777216 + avoid_flush_during_shutdown=false + writable_file_max_buffer_size=1048576 + max_subcompactions=1 + base_background_compactions=-1 + max_background_compactions=-1 + max_background_jobs=2 + file_checksum_gen_factory=nullptr + db_host_id=__hostname__ + bgerror_resume_retry_interval=1000000 + best_efforts_recovery=false + avoid_unnecessary_blocking_io=false + two_write_queues=false + atomic_flush=false + preserve_deletes=false + allow_ingest_behind=false + lowest_used_cache_tier=kNonVolatileBlockTier + avoid_flush_during_recovery=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + max_bgerror_resume_count=2147483647 + write_thread_slow_yield_usec=3 + allow_concurrent_memtable_write=true + WAL_ttl_seconds=0 + manual_wal_flush=false + dump_malloc_stats=false + wal_recovery_mode=kPointInTimeRecovery + log_file_time_to_roll=86400 + enable_write_thread_adaptive_yield=true + recycle_log_file_num=0 + table_cache_numshardbits=6 + max_file_opening_threads=16 + allow_data_in_errors=false + rate_limiter=nullptr + use_fsync=false + unordered_write=false + fail_if_options_file_error=false + random_access_max_buffer_size=1048576 + new_table_reader_for_compaction_inputs=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + persist_stats_to_disk=false + track_and_verify_wals_in_manifest=false + enable_pipelined_write=false + flush_verify_memtable_count=true + log_readahead_size=0 + is_fd_close_on_exec=true + WAL_size_limit_MB=0 + experimental_mempurge_threshold=0.000000 + write_dbid_to_manifest=false + use_adaptive_mutex=false + error_if_exists=false + write_thread_max_yield_usec=100 + enable_thread_tracking=false + db_write_buffer_size=0 + create_missing_column_families=false + paranoid_checks=true + create_if_missing=true + wal_filter=nullptr + max_manifest_file_size=1073741824 + allow_2pc=true + use_direct_io_for_flush_and_compaction=false + manifest_preallocation_size=4194304 + use_direct_reads=false + allow_fallocate=true + max_write_batch_group_size_bytes=1048576 + keep_log_file_num=30 + allow_mmap_reads=false + max_log_file_size=0 + allow_mmap_writes=false + advise_random_on_open=true + + +[CFOptions "default"] + compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + bottommost_compression=kDisableCompressionOption + enable_blob_garbage_collection=false + blob_file_size=268435456 + sample_for_compression=0 + periodic_compaction_seconds=0 + ttl=2592000 + blob_garbage_collection_age_cutoff=0.250000 + compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;} + compression=kSnappyCompression + max_sequential_skip_in_iterations=8 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + max_bytes_for_level_multiplier=10.000000 + max_bytes_for_level_base=268435456 + memtable_whole_key_filtering=false + max_successive_merges=0 + blob_compaction_readahead_size=0 + inplace_update_num_locks=10000 + arena_block_size=1048576 + target_file_size_multiplier=1 + prefix_extractor=nullptr + max_write_buffer_number=2 + blob_compression_type=kNoCompression + level0_stop_writes_trigger=36 + level0_slowdown_writes_trigger=20 + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + level0_file_num_compaction_trigger=4 + write_buffer_size=67108864 + memtable_huge_page_size=0 + max_compaction_bytes=1677721600 + min_blob_size=0 + bottommost_compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + target_file_size_base=67108864 + soft_pending_compaction_bytes_limit=68719476736 + enable_blob_files=false + paranoid_file_checks=false + check_flush_compaction_key_order=true + blob_garbage_collection_force_threshold=1.000000 + disable_auto_compactions=true + memtable_prefix_bloom_size_ratio=0.000000 + report_bg_io_stats=false + compaction_pri=kMinOverlappingRatio + compaction_filter_factory=nullptr + comparator=leveldb.BytewiseComparator + sst_partitioner_factory=nullptr + bloom_locality=0 + compaction_style=kCompactionStyleLevel + min_write_buffer_number_to_merge=1 + max_write_buffer_size_to_maintain=134217728 + max_write_buffer_number_to_maintain=0 + merge_operator=nullptr + memtable_factory=SkipListFactory + memtable_insert_with_hint_prefix_extractor=nullptr + num_levels=7 + force_consistency_checks=true + optimize_filters_for_hits=false + compaction_filter=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + table_factory=BlockBasedTable + +[TableOptions/BlockBasedTable "default"] + pin_top_level_index_and_filter=true + block_align=false + read_amp_bytes_per_bit=0 + verify_compression=false + enable_index_compression=true + reserve_table_builder_memory=false + whole_key_filtering=true + max_auto_readahead_size=262144 + optimize_filters_for_memory=false + index_block_restart_interval=1 + prepopulate_block_cache=kDisable + block_restart_interval=16 + block_size=4096 + format_version=5 + partition_filters=false + block_size_deviation=10 + no_block_cache=false + checksum=kCRC32c + data_block_hash_table_util_ratio=0.750000 + index_shortening=kShortenSeparators + data_block_index_type=kDataBlockBinarySearch + hash_index_allow_collision=true + filter_policy=nullptr + metadata_block_size=4096 + index_type=kBinarySearch + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/devmetadata/OPTIONS-000009 b/devmetadata/OPTIONS-000009 new file mode 100644 index 0000000000000..b9d0340f85625 --- /dev/null +++ b/devmetadata/OPTIONS-000009 @@ -0,0 +1,190 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.29.4 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + wal_bytes_per_sync=0 + bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + stats_dump_period_sec=600 + stats_persist_period_sec=600 + delete_obsolete_files_period_micros=21600000000 + max_total_wal_size=0 + strict_bytes_per_sync=false + delayed_write_rate=16777216 + avoid_flush_during_shutdown=false + writable_file_max_buffer_size=1048576 + max_subcompactions=1 + base_background_compactions=-1 + max_background_compactions=-1 + max_background_jobs=2 + file_checksum_gen_factory=nullptr + db_host_id=__hostname__ + bgerror_resume_retry_interval=1000000 + best_efforts_recovery=false + avoid_unnecessary_blocking_io=false + two_write_queues=false + atomic_flush=false + preserve_deletes=false + allow_ingest_behind=false + lowest_used_cache_tier=kNonVolatileBlockTier + avoid_flush_during_recovery=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + max_bgerror_resume_count=2147483647 + write_thread_slow_yield_usec=3 + allow_concurrent_memtable_write=true + WAL_ttl_seconds=0 + manual_wal_flush=false + dump_malloc_stats=false + wal_recovery_mode=kPointInTimeRecovery + log_file_time_to_roll=86400 + enable_write_thread_adaptive_yield=true + recycle_log_file_num=0 + table_cache_numshardbits=6 + max_file_opening_threads=16 + allow_data_in_errors=false + rate_limiter=nullptr + use_fsync=false + unordered_write=false + fail_if_options_file_error=false + random_access_max_buffer_size=1048576 + new_table_reader_for_compaction_inputs=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + persist_stats_to_disk=false + track_and_verify_wals_in_manifest=false + enable_pipelined_write=false + flush_verify_memtable_count=true + log_readahead_size=0 + is_fd_close_on_exec=true + WAL_size_limit_MB=0 + experimental_mempurge_threshold=0.000000 + write_dbid_to_manifest=false + use_adaptive_mutex=false + error_if_exists=false + write_thread_max_yield_usec=100 + enable_thread_tracking=false + db_write_buffer_size=0 + create_missing_column_families=false + paranoid_checks=true + create_if_missing=true + wal_filter=nullptr + max_manifest_file_size=1073741824 + allow_2pc=true + use_direct_io_for_flush_and_compaction=false + manifest_preallocation_size=4194304 + use_direct_reads=false + allow_fallocate=true + max_write_batch_group_size_bytes=1048576 + keep_log_file_num=30 + allow_mmap_reads=false + max_log_file_size=0 + allow_mmap_writes=false + advise_random_on_open=true + + +[CFOptions "default"] + compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + bottommost_compression=kDisableCompressionOption + enable_blob_garbage_collection=false + blob_file_size=268435456 + sample_for_compression=0 + periodic_compaction_seconds=0 + ttl=2592000 + blob_garbage_collection_age_cutoff=0.250000 + compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;} + compression=kSnappyCompression + max_sequential_skip_in_iterations=8 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + max_bytes_for_level_multiplier=10.000000 + max_bytes_for_level_base=268435456 + memtable_whole_key_filtering=false + max_successive_merges=0 + blob_compaction_readahead_size=0 + inplace_update_num_locks=10000 + arena_block_size=1048576 + target_file_size_multiplier=1 + prefix_extractor=nullptr + max_write_buffer_number=2 + blob_compression_type=kNoCompression + level0_stop_writes_trigger=36 + level0_slowdown_writes_trigger=20 + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + level0_file_num_compaction_trigger=4 + write_buffer_size=67108864 + memtable_huge_page_size=0 + max_compaction_bytes=1677721600 + min_blob_size=0 + bottommost_compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + target_file_size_base=67108864 + soft_pending_compaction_bytes_limit=68719476736 + enable_blob_files=false + paranoid_file_checks=false + check_flush_compaction_key_order=true + blob_garbage_collection_force_threshold=1.000000 + disable_auto_compactions=false + memtable_prefix_bloom_size_ratio=0.000000 + report_bg_io_stats=false + compaction_pri=kMinOverlappingRatio + compaction_filter_factory=nullptr + comparator=leveldb.BytewiseComparator + sst_partitioner_factory=nullptr + bloom_locality=0 + compaction_style=kCompactionStyleLevel + min_write_buffer_number_to_merge=1 + max_write_buffer_size_to_maintain=134217728 + max_write_buffer_number_to_maintain=0 + merge_operator=nullptr + memtable_factory=SkipListFactory + memtable_insert_with_hint_prefix_extractor=nullptr + num_levels=7 + force_consistency_checks=true + optimize_filters_for_hits=false + compaction_filter=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + table_factory=BlockBasedTable + +[TableOptions/BlockBasedTable "default"] + pin_top_level_index_and_filter=true + block_align=false + read_amp_bytes_per_bit=0 + verify_compression=false + enable_index_compression=true + reserve_table_builder_memory=false + whole_key_filtering=true + max_auto_readahead_size=262144 + optimize_filters_for_memory=false + index_block_restart_interval=1 + prepopulate_block_cache=kDisable + block_restart_interval=16 + block_size=4096 + format_version=5 + partition_filters=false + block_size_deviation=10 + no_block_cache=false + checksum=kCRC32c + data_block_hash_table_util_ratio=0.750000 + index_shortening=kShortenSeparators + data_block_index_type=kDataBlockBinarySearch + hash_index_allow_collision=true + filter_policy=nullptr + metadata_block_size=4096 + index_type=kBinarySearch + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/docker/pom.xml b/docker/pom.xml index 3ce4c6bac6f8e..4d4b7b8888694 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -58,7 +58,7 @@ pulsar - pulsar-all + diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 3746fe1c26c2e..4b5ffda353b0c 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -21,6 +21,7 @@ FROM busybox as pulsar ARG PULSAR_TARBALL +# 21:28 ADD ${PULSAR_TARBALL} / RUN mv /apache-pulsar-* /pulsar @@ -30,9 +31,9 @@ COPY scripts/apply-config-from-env.py /pulsar/bin COPY scripts/apply-config-from-env-with-prefix.py /pulsar/bin COPY scripts/gen-yml-from-env.py /pulsar/bin COPY scripts/generate-zookeeper-config.sh /pulsar/bin -COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin +#COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin COPY scripts/watch-znode.py /pulsar/bin -COPY scripts/install-pulsar-client.sh /pulsar/bin +#COPY scripts/install-pulsar-client.sh /pulsar/bin # The final image needs to give the root group sufficient permission for Pulsar components # to write to specific directories within /pulsar @@ -59,6 +60,19 @@ RUN sed -i "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirr && echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Retries "3";' > /etc/apt/apt.conf.d/99timeout_and_retries \ && apt-get update \ && apt-get -y dist-upgrade \ +<<<<<<< HEAD +<<<<<<< HEAD + && apt-get -y install --no-install-recommends openjdk-11-jdk-headless \ +# netcat dnsutils less procps iputils-ping \ +# python3 python3-yaml python3-kazoo python3-pip \ +# curl ca-certificates \ +======= + && apt-get -y install --no-install-recommends openjdk-17-jdk-headless netcat dnsutils less procps iputils-ping \ + python3 python3-kazoo python3-pip \ + curl ca-certificates \ +>>>>>>> e6b12c64b043903eb5ff2dc5186fe8030f157cfc + && apt-get -y --purge autoremove \ +======= && apt-get -y install --no-install-recommends netcat dnsutils less procps iputils-ping \ python3 python3-kazoo python3-pip \ curl ca-certificates wget apt-transport-https @@ -75,11 +89,16 @@ RUN mkdir -p /etc/apt/keyrings \ # Cleanup apt RUN apt-get -y --purge autoremove \ +>>>>>>> origin/master && apt-get autoclean \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +<<<<<<< HEAD +#RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +======= RUN pip3 install pyyaml==5.4.1 +>>>>>>> e6b12c64b043903eb5ff2dc5186fe8030f157cfc # Pulsar currently writes to the below directories, assuming the default configuration. # Note that number 4 is the reason that pulsar components need write access to the /pulsar directory. @@ -89,7 +108,18 @@ RUN pip3 install pyyaml==5.4.1 # 4. /pulsar - hadoop writes to this directory RUN mkdir /pulsar && chmod g+w /pulsar +<<<<<<< HEAD +<<<<<<< HEAD +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security +#ADD target/python-client/ /pulsar/pulsar-client +======= +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-17-openjdk-amd64/conf/security/java.security +======= +>>>>>>> origin/master ADD target/python-client/ /pulsar/pulsar-client +>>>>>>> e6b12c64b043903eb5ff2dc5186fe8030f157cfc ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE @@ -97,7 +127,7 @@ COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar # This script is intentionally run as the root user to make the dependencies available for all UIDs. -RUN /pulsar/bin/install-pulsar-client.sh +#RUN /pulsar/bin/install-pulsar-client.sh # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. USER 10000 diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index a458cea39763a..ef83be3974909 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -31,9 +31,14 @@ pom +<<<<<<< HEAD + true + true +======= x86_64 false false +>>>>>>> origin/master diff --git a/logFungo b/logFungo new file mode 100755 index 0000000000000..88e3409dffc06 Binary files /dev/null and b/logFungo differ diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 71bc8cad6f0b3..43586b98d66b7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -80,7 +80,7 @@ import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.common.util.Retries; -import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.*; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; @@ -92,11 +92,6 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedger; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.InvalidCursorPositionException; @@ -109,9 +104,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; -import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.WaitingEntryCallBack; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; import org.apache.bookkeeper.mledger.impl.cache.EntryCache; diff --git a/metadata/000009.sst b/metadata/000009.sst new file mode 100644 index 0000000000000..8502d42a3bb50 Binary files /dev/null and b/metadata/000009.sst differ diff --git a/metadata/CURRENT b/metadata/CURRENT new file mode 100644 index 0000000000000..3051f81a611ab --- /dev/null +++ b/metadata/CURRENT @@ -0,0 +1 @@ +MANIFEST-000010 diff --git a/metadata/IDENTITY b/metadata/IDENTITY new file mode 100644 index 0000000000000..5eb8f32204dbd --- /dev/null +++ b/metadata/IDENTITY @@ -0,0 +1 @@ +0035a3e9-f890-41f4-ae60-51a8381c6dfa diff --git a/metadata/LOCK b/metadata/LOCK new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata/LOG b/metadata/LOG new file mode 100644 index 0000000000000..93d4a651a8b08 --- /dev/null +++ b/metadata/LOG @@ -0,0 +1,337 @@ +2022/06/22-11:41:09.747094 7fdb73c14700 RocksDB version: 6.16.4 +2022/06/22-11:41:09.747149 7fdb73c14700 Git sha rocksdb_build_git_sha: +2022/06/22-11:41:09.747153 7fdb73c14700 Compile date Mar 31 2021 +2022/06/22-11:41:09.747220 7fdb73c14700 DB SUMMARY +2022/06/22-11:41:09.747223 7fdb73c14700 DB Session ID: IXUVLCYN8D48AMTQABMY +2022/06/22-11:41:09.747257 7fdb73c14700 CURRENT file: CURRENT +2022/06/22-11:41:09.747260 7fdb73c14700 IDENTITY file: IDENTITY +2022/06/22-11:41:09.747265 7fdb73c14700 MANIFEST file: MANIFEST-000003 size: 70 Bytes +2022/06/22-11:41:09.747269 7fdb73c14700 SST files in /home/lan/repo/git/pulsar/metadata dir, Total Num: 0, files: +2022/06/22-11:41:09.747271 7fdb73c14700 Write Ahead Log file in /home/lan/repo/git/pulsar/metadata: 000004.log size: 112 ; +2022/06/22-11:41:09.747276 7fdb73c14700 Options.error_if_exists: 0 +2022/06/22-11:41:09.747278 7fdb73c14700 Options.create_if_missing: 1 +2022/06/22-11:41:09.747280 7fdb73c14700 Options.paranoid_checks: 1 +2022/06/22-11:41:09.747282 7fdb73c14700 Options.track_and_verify_wals_in_manifest: 0 +2022/06/22-11:41:09.747285 7fdb73c14700 Options.env: 0x7fdaefdbbc60 +2022/06/22-11:41:09.747287 7fdb73c14700 Options.fs: Posix File System +2022/06/22-11:41:09.747291 7fdb73c14700 Options.info_log: 0x7fdb6d393340 +2022/06/22-11:41:09.747294 7fdb73c14700 Options.max_file_opening_threads: 16 +2022/06/22-11:41:09.747296 7fdb73c14700 Options.statistics: (nil) +2022/06/22-11:41:09.747298 7fdb73c14700 Options.use_fsync: 0 +2022/06/22-11:41:09.747300 7fdb73c14700 Options.max_log_file_size: 0 +2022/06/22-11:41:09.747302 7fdb73c14700 Options.max_manifest_file_size: 1073741824 +2022/06/22-11:41:09.747305 7fdb73c14700 Options.log_file_time_to_roll: 86400 +2022/06/22-11:41:09.747307 7fdb73c14700 Options.keep_log_file_num: 30 +2022/06/22-11:41:09.747309 7fdb73c14700 Options.recycle_log_file_num: 0 +2022/06/22-11:41:09.747311 7fdb73c14700 Options.allow_fallocate: 1 +2022/06/22-11:41:09.747313 7fdb73c14700 Options.allow_mmap_reads: 0 +2022/06/22-11:41:09.747315 7fdb73c14700 Options.allow_mmap_writes: 0 +2022/06/22-11:41:09.747317 7fdb73c14700 Options.use_direct_reads: 0 +2022/06/22-11:41:09.747319 7fdb73c14700 Options.use_direct_io_for_flush_and_compaction: 0 +2022/06/22-11:41:09.747321 7fdb73c14700 Options.create_missing_column_families: 0 +2022/06/22-11:41:09.747324 7fdb73c14700 Options.db_log_dir: +2022/06/22-11:41:09.747326 7fdb73c14700 Options.wal_dir: /home/lan/repo/git/pulsar/metadata +2022/06/22-11:41:09.747328 7fdb73c14700 Options.table_cache_numshardbits: 6 +2022/06/22-11:41:09.747330 7fdb73c14700 Options.WAL_ttl_seconds: 0 +2022/06/22-11:41:09.747332 7fdb73c14700 Options.WAL_size_limit_MB: 0 +2022/06/22-11:41:09.747334 7fdb73c14700 Options.max_write_batch_group_size_bytes: 1048576 +2022/06/22-11:41:09.747336 7fdb73c14700 Options.manifest_preallocation_size: 4194304 +2022/06/22-11:41:09.747338 7fdb73c14700 Options.is_fd_close_on_exec: 1 +2022/06/22-11:41:09.747340 7fdb73c14700 Options.advise_random_on_open: 1 +2022/06/22-11:41:09.747342 7fdb73c14700 Options.db_write_buffer_size: 0 +2022/06/22-11:41:09.747344 7fdb73c14700 Options.write_buffer_manager: 0x7fdb6d393830 +2022/06/22-11:41:09.747347 7fdb73c14700 Options.access_hint_on_compaction_start: 1 +2022/06/22-11:41:09.747349 7fdb73c14700 Options.new_table_reader_for_compaction_inputs: 0 +2022/06/22-11:41:09.747351 7fdb73c14700 Options.random_access_max_buffer_size: 1048576 +2022/06/22-11:41:09.747353 7fdb73c14700 Options.use_adaptive_mutex: 0 +2022/06/22-11:41:09.747370 7fdb73c14700 Options.rate_limiter: (nil) +2022/06/22-11:41:09.747374 7fdb73c14700 Options.sst_file_manager.rate_bytes_per_sec: 0 +2022/06/22-11:41:09.747376 7fdb73c14700 Options.wal_recovery_mode: 2 +2022/06/22-11:41:09.747378 7fdb73c14700 Options.enable_thread_tracking: 0 +2022/06/22-11:41:09.747380 7fdb73c14700 Options.enable_pipelined_write: 0 +2022/06/22-11:41:09.747382 7fdb73c14700 Options.unordered_write: 0 +2022/06/22-11:41:09.747384 7fdb73c14700 Options.allow_concurrent_memtable_write: 1 +2022/06/22-11:41:09.747386 7fdb73c14700 Options.enable_write_thread_adaptive_yield: 1 +2022/06/22-11:41:09.747388 7fdb73c14700 Options.write_thread_max_yield_usec: 100 +2022/06/22-11:41:09.747390 7fdb73c14700 Options.write_thread_slow_yield_usec: 3 +2022/06/22-11:41:09.747392 7fdb73c14700 Options.row_cache: None +2022/06/22-11:41:09.747395 7fdb73c14700 Options.wal_filter: None +2022/06/22-11:41:09.747397 7fdb73c14700 Options.avoid_flush_during_recovery: 0 +2022/06/22-11:41:09.747399 7fdb73c14700 Options.allow_ingest_behind: 0 +2022/06/22-11:41:09.747401 7fdb73c14700 Options.preserve_deletes: 0 +2022/06/22-11:41:09.747403 7fdb73c14700 Options.two_write_queues: 0 +2022/06/22-11:41:09.747405 7fdb73c14700 Options.manual_wal_flush: 0 +2022/06/22-11:41:09.747407 7fdb73c14700 Options.atomic_flush: 0 +2022/06/22-11:41:09.747409 7fdb73c14700 Options.avoid_unnecessary_blocking_io: 0 +2022/06/22-11:41:09.747411 7fdb73c14700 Options.persist_stats_to_disk: 0 +2022/06/22-11:41:09.747413 7fdb73c14700 Options.write_dbid_to_manifest: 0 +2022/06/22-11:41:09.747415 7fdb73c14700 Options.log_readahead_size: 0 +2022/06/22-11:41:09.747418 7fdb73c14700 Options.file_checksum_gen_factory: Unknown +2022/06/22-11:41:09.747420 7fdb73c14700 Options.best_efforts_recovery: 0 +2022/06/22-11:41:09.747422 7fdb73c14700 Options.max_bgerror_resume_count: 2147483647 +2022/06/22-11:41:09.747424 7fdb73c14700 Options.bgerror_resume_retry_interval: 1000000 +2022/06/22-11:41:09.747426 7fdb73c14700 Options.allow_data_in_errors: 0 +2022/06/22-11:41:09.747428 7fdb73c14700 Options.db_host_id: __hostname__ +2022/06/22-11:41:09.747431 7fdb73c14700 Options.max_background_jobs: 2 +2022/06/22-11:41:09.747434 7fdb73c14700 Options.max_background_compactions: -1 +2022/06/22-11:41:09.747436 7fdb73c14700 Options.max_subcompactions: 1 +2022/06/22-11:41:09.747438 7fdb73c14700 Options.avoid_flush_during_shutdown: 0 +2022/06/22-11:41:09.747440 7fdb73c14700 Options.writable_file_max_buffer_size: 1048576 +2022/06/22-11:41:09.747442 7fdb73c14700 Options.delayed_write_rate : 16777216 +2022/06/22-11:41:09.747445 7fdb73c14700 Options.max_total_wal_size: 0 +2022/06/22-11:41:09.747450 7fdb73c14700 Options.delete_obsolete_files_period_micros: 21600000000 +2022/06/22-11:41:09.747453 7fdb73c14700 Options.stats_dump_period_sec: 600 +2022/06/22-11:41:09.747455 7fdb73c14700 Options.stats_persist_period_sec: 600 +2022/06/22-11:41:09.747457 7fdb73c14700 Options.stats_history_buffer_size: 1048576 +2022/06/22-11:41:09.747459 7fdb73c14700 Options.max_open_files: -1 +2022/06/22-11:41:09.747461 7fdb73c14700 Options.bytes_per_sync: 0 +2022/06/22-11:41:09.747463 7fdb73c14700 Options.wal_bytes_per_sync: 0 +2022/06/22-11:41:09.747465 7fdb73c14700 Options.strict_bytes_per_sync: 0 +2022/06/22-11:41:09.747467 7fdb73c14700 Options.compaction_readahead_size: 0 +2022/06/22-11:41:09.747469 7fdb73c14700 Options.max_background_flushes: -1 +2022/06/22-11:41:09.747479 7fdb73c14700 Compression algorithms supported: +2022/06/22-11:41:09.747482 7fdb73c14700 kZSTDNotFinalCompression supported: 1 +2022/06/22-11:41:09.747490 7fdb73c14700 kZSTD supported: 1 +2022/06/22-11:41:09.747492 7fdb73c14700 kXpressCompression supported: 0 +2022/06/22-11:41:09.747494 7fdb73c14700 kLZ4HCCompression supported: 1 +2022/06/22-11:41:09.747496 7fdb73c14700 kLZ4Compression supported: 1 +2022/06/22-11:41:09.747498 7fdb73c14700 kBZip2Compression supported: 1 +2022/06/22-11:41:09.747500 7fdb73c14700 kZlibCompression supported: 1 +2022/06/22-11:41:09.747502 7fdb73c14700 kSnappyCompression supported: 1 +2022/06/22-11:41:09.747508 7fdb73c14700 Fast CRC32 supported: Not supported on x86 +2022/06/22-11:41:09.747619 7fdb73c14700 [/version_set.cc:4757] Recovering from manifest file: /home/lan/repo/git/pulsar/metadata/MANIFEST-000003 +2022/06/22-11:41:09.747828 7fdb73c14700 [/column_family.cc:595] --------------- Options for column family [default]: +2022/06/22-11:41:09.747834 7fdb73c14700 Options.comparator: leveldb.BytewiseComparator +2022/06/22-11:41:09.747836 7fdb73c14700 Options.merge_operator: None +2022/06/22-11:41:09.747838 7fdb73c14700 Options.compaction_filter: None +2022/06/22-11:41:09.747840 7fdb73c14700 Options.compaction_filter_factory: None +2022/06/22-11:41:09.747842 7fdb73c14700 Options.sst_partitioner_factory: None +2022/06/22-11:41:09.747845 7fdb73c14700 Options.memtable_factory: SkipListFactory +2022/06/22-11:41:09.747850 7fdb73c14700 Options.table_factory: BlockBasedTable +2022/06/22-11:41:09.747885 7fdb73c14700 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fdb6d3909b0) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 1 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + data_block_index_type: 0 + index_shortening: 1 + data_block_hash_table_util_ratio: 0.750000 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7fdb6d3a3d80 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: (nil) + persistent_cache: (nil) + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 4 + enable_index_compression: 1 + block_align: 0 +2022/06/22-11:41:09.747889 7fdb73c14700 Options.write_buffer_size: 67108864 +2022/06/22-11:41:09.747891 7fdb73c14700 Options.max_write_buffer_number: 2 +2022/06/22-11:41:09.747894 7fdb73c14700 Options.compression: Snappy +2022/06/22-11:41:09.747896 7fdb73c14700 Options.bottommost_compression: Disabled +2022/06/22-11:41:09.747898 7fdb73c14700 Options.prefix_extractor: nullptr +2022/06/22-11:41:09.747900 7fdb73c14700 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2022/06/22-11:41:09.747902 7fdb73c14700 Options.num_levels: 7 +2022/06/22-11:41:09.747904 7fdb73c14700 Options.min_write_buffer_number_to_merge: 1 +2022/06/22-11:41:09.747907 7fdb73c14700 Options.max_write_buffer_number_to_maintain: 0 +2022/06/22-11:41:09.747909 7fdb73c14700 Options.max_write_buffer_size_to_maintain: 134217728 +2022/06/22-11:41:09.747911 7fdb73c14700 Options.bottommost_compression_opts.window_bits: -14 +2022/06/22-11:41:09.747913 7fdb73c14700 Options.bottommost_compression_opts.level: 32767 +2022/06/22-11:41:09.747915 7fdb73c14700 Options.bottommost_compression_opts.strategy: 0 +2022/06/22-11:41:09.747917 7fdb73c14700 Options.bottommost_compression_opts.max_dict_bytes: 0 +2022/06/22-11:41:09.747919 7fdb73c14700 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2022/06/22-11:41:09.747933 7fdb73c14700 Options.bottommost_compression_opts.parallel_threads: 1 +2022/06/22-11:41:09.747936 7fdb73c14700 Options.bottommost_compression_opts.enabled: false +2022/06/22-11:41:09.747938 7fdb73c14700 Options.compression_opts.window_bits: -14 +2022/06/22-11:41:09.747940 7fdb73c14700 Options.compression_opts.level: 32767 +2022/06/22-11:41:09.747942 7fdb73c14700 Options.compression_opts.strategy: 0 +2022/06/22-11:41:09.747944 7fdb73c14700 Options.compression_opts.max_dict_bytes: 0 +2022/06/22-11:41:09.747946 7fdb73c14700 Options.compression_opts.zstd_max_train_bytes: 0 +2022/06/22-11:41:09.747948 7fdb73c14700 Options.compression_opts.parallel_threads: 1 +2022/06/22-11:41:09.747950 7fdb73c14700 Options.compression_opts.enabled: false +2022/06/22-11:41:09.747952 7fdb73c14700 Options.level0_file_num_compaction_trigger: 4 +2022/06/22-11:41:09.747954 7fdb73c14700 Options.level0_slowdown_writes_trigger: 20 +2022/06/22-11:41:09.747956 7fdb73c14700 Options.level0_stop_writes_trigger: 36 +2022/06/22-11:41:09.747958 7fdb73c14700 Options.target_file_size_base: 67108864 +2022/06/22-11:41:09.747960 7fdb73c14700 Options.target_file_size_multiplier: 1 +2022/06/22-11:41:09.747962 7fdb73c14700 Options.max_bytes_for_level_base: 268435456 +2022/06/22-11:41:09.747964 7fdb73c14700 Options.level_compaction_dynamic_level_bytes: 0 +2022/06/22-11:41:09.747968 7fdb73c14700 Options.max_bytes_for_level_multiplier: 10.000000 +2022/06/22-11:41:09.747970 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2022/06/22-11:41:09.747975 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2022/06/22-11:41:09.747977 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2022/06/22-11:41:09.747980 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2022/06/22-11:41:09.747982 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2022/06/22-11:41:09.747984 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2022/06/22-11:41:09.747986 7fdb73c14700 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2022/06/22-11:41:09.747988 7fdb73c14700 Options.max_sequential_skip_in_iterations: 8 +2022/06/22-11:41:09.747990 7fdb73c14700 Options.max_compaction_bytes: 1677721600 +2022/06/22-11:41:09.747992 7fdb73c14700 Options.arena_block_size: 8388608 +2022/06/22-11:41:09.747995 7fdb73c14700 Options.soft_pending_compaction_bytes_limit: 68719476736 +2022/06/22-11:41:09.747997 7fdb73c14700 Options.hard_pending_compaction_bytes_limit: 274877906944 +2022/06/22-11:41:09.747999 7fdb73c14700 Options.rate_limit_delay_max_milliseconds: 100 +2022/06/22-11:41:09.748001 7fdb73c14700 Options.disable_auto_compactions: 1 +2022/06/22-11:41:09.748003 7fdb73c14700 Options.compaction_style: kCompactionStyleLevel +2022/06/22-11:41:09.748006 7fdb73c14700 Options.compaction_pri: kMinOverlappingRatio +2022/06/22-11:41:09.748008 7fdb73c14700 Options.compaction_options_universal.size_ratio: 1 +2022/06/22-11:41:09.748010 7fdb73c14700 Options.compaction_options_universal.min_merge_width: 2 +2022/06/22-11:41:09.748012 7fdb73c14700 Options.compaction_options_universal.max_merge_width: 4294967295 +2022/06/22-11:41:09.748014 7fdb73c14700 Options.compaction_options_universal.max_size_amplification_percent: 200 +2022/06/22-11:41:09.748016 7fdb73c14700 Options.compaction_options_universal.compression_size_percent: -1 +2022/06/22-11:41:09.748018 7fdb73c14700 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2022/06/22-11:41:09.748020 7fdb73c14700 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2022/06/22-11:41:09.748022 7fdb73c14700 Options.compaction_options_fifo.allow_compaction: 0 +2022/06/22-11:41:09.748028 7fdb73c14700 Options.table_properties_collectors: +2022/06/22-11:41:09.748038 7fdb73c14700 Options.inplace_update_support: 0 +2022/06/22-11:41:09.748041 7fdb73c14700 Options.inplace_update_num_locks: 10000 +2022/06/22-11:41:09.748044 7fdb73c14700 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2022/06/22-11:41:09.748046 7fdb73c14700 Options.memtable_whole_key_filtering: 0 +2022/06/22-11:41:09.748048 7fdb73c14700 Options.memtable_huge_page_size: 0 +2022/06/22-11:41:09.748050 7fdb73c14700 Options.bloom_locality: 0 +2022/06/22-11:41:09.748052 7fdb73c14700 Options.max_successive_merges: 0 +2022/06/22-11:41:09.748056 7fdb73c14700 Options.optimize_filters_for_hits: 0 +2022/06/22-11:41:09.748059 7fdb73c14700 Options.paranoid_file_checks: 0 +2022/06/22-11:41:09.748061 7fdb73c14700 Options.force_consistency_checks: 1 +2022/06/22-11:41:09.748063 7fdb73c14700 Options.report_bg_io_stats: 0 +2022/06/22-11:41:09.748065 7fdb73c14700 Options.ttl: 2592000 +2022/06/22-11:41:09.748067 7fdb73c14700 Options.periodic_compaction_seconds: 0 +2022/06/22-11:41:09.748069 7fdb73c14700 Options.enable_blob_files: false +2022/06/22-11:41:09.748071 7fdb73c14700 Options.min_blob_size: 0 +2022/06/22-11:41:09.748074 7fdb73c14700 Options.blob_file_size: 268435456 +2022/06/22-11:41:09.748076 7fdb73c14700 Options.blob_compression_type: NoCompression +2022/06/22-11:41:09.748078 7fdb73c14700 Options.enable_blob_garbage_collection: false +2022/06/22-11:41:09.748081 7fdb73c14700 Options.blob_garbage_collection_age_cutoff: 0.250000 +2022/06/22-11:41:09.752342 7fdb73c14700 [/version_set.cc:4805] Recovered from manifest file:/home/lan/repo/git/pulsar/metadata/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 8, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2022/06/22-11:41:09.752352 7fdb73c14700 [/version_set.cc:4814] Column family [default] (ID 0), log number is 0 +2022/06/22-11:41:09.752478 7fdb73c14700 [/version_set.cc:4111] Creating manifest 8 +2022/06/22-11:41:09.765444 7fdb73c14700 EVENT_LOG_v1 {"time_micros": 1655869269765426, "job": 1, "event": "recovery_started", "wal_files": [4]} +2022/06/22-11:41:09.765459 7fdb73c14700 [/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 +2022/06/22-11:41:09.767558 7fdb73c14700 EVENT_LOG_v1 {"time_micros": 1655869269767513, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 9, "file_size": 935, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 57, "index_size": 42, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 0, "raw_key_size": 33, "raw_average_key_size": 33, "raw_value_size": 8, "raw_average_value_size": 8, "num_data_blocks": 1, "num_entries": 1, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "nullptr", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "Snappy", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1655869269, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0035a3e9-f890-41f4-ae60-51a8381c6dfa", "db_session_id": "IXUVLCYN8D48AMTQABMY"}} +2022/06/22-11:41:09.767750 7fdb73c14700 [/version_set.cc:4111] Creating manifest 10 +2022/06/22-11:41:09.778764 7fdb73c14700 EVENT_LOG_v1 {"time_micros": 1655869269778755, "job": 1, "event": "recovery_finished"} +2022/06/22-11:41:09.788732 7fdb73c14700 [/db_impl/db_impl_open.cc:1705] SstFileManager instance 0x7fdb6d238000 +2022/06/22-11:41:09.788800 7fdb73c14700 DB pointer 0x7fdb6d395f00 +2022/06/22-11:41:09.788887 7fdb73c14700 [WARN] [ilities/transactions/pessimistic_transaction_db.cc:244] Transaction write_policy is 0 +2022/06/22-11:41:09.789276 7fdae2ff5700 [/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- +2022/06/22-11:41:09.789315 7fdae2ff5700 [/db_impl/db_impl.cc:903] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + L0 1/0 0.91 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.4 0.00 0.00 1 0.002 0 0 + Sum 1/0 0.91 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.4 0.00 0.00 1 0.002 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.4 0.00 0.00 1 0.002 0 0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.4 0.00 0.00 1 0.002 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + L0 1/0 0.91 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.4 0.00 0.00 1 0.002 0 0 + Sum 1/0 0.91 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.4 0.00 0.00 1 0.002 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.4 0.00 0.00 1 0.002 0 0 +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** +2022/06/22-11:41:09.797737 7fdb73c14700 [/db_impl/db_impl.cc:1006] SetOptions() on column family [default], inputs: +2022/06/22-11:41:09.797746 7fdb73c14700 [/db_impl/db_impl.cc:1009] disable_auto_compactions: false +2022/06/22-11:41:09.797749 7fdb73c14700 [/db_impl/db_impl.cc:1013] [default] SetOptions() succeeded +2022/06/22-11:41:09.797754 7fdb73c14700 [tions/cf_options.cc:910] write_buffer_size: 67108864 +2022/06/22-11:41:09.797756 7fdb73c14700 [tions/cf_options.cc:912] max_write_buffer_number: 2 +2022/06/22-11:41:09.797758 7fdb73c14700 [tions/cf_options.cc:915] arena_block_size: 8388608 +2022/06/22-11:41:09.797759 7fdb73c14700 [tions/cf_options.cc:917] memtable_prefix_bloom_ratio: 0.000000 +2022/06/22-11:41:09.797762 7fdb73c14700 [tions/cf_options.cc:919] memtable_whole_key_filtering: 0 +2022/06/22-11:41:09.797764 7fdb73c14700 [tions/cf_options.cc:922] memtable_huge_page_size: 0 +2022/06/22-11:41:09.797766 7fdb73c14700 [tions/cf_options.cc:925] max_successive_merges: 0 +2022/06/22-11:41:09.797768 7fdb73c14700 [tions/cf_options.cc:928] inplace_update_num_locks: 10000 +2022/06/22-11:41:09.797770 7fdb73c14700 [tions/cf_options.cc:931] prefix_extractor: nullptr +2022/06/22-11:41:09.797772 7fdb73c14700 [tions/cf_options.cc:933] disable_auto_compactions: 0 +2022/06/22-11:41:09.797774 7fdb73c14700 [tions/cf_options.cc:935] soft_pending_compaction_bytes_limit: 68719476736 +2022/06/22-11:41:09.797776 7fdb73c14700 [tions/cf_options.cc:937] hard_pending_compaction_bytes_limit: 274877906944 +2022/06/22-11:41:09.797778 7fdb73c14700 [tions/cf_options.cc:939] level0_file_num_compaction_trigger: 4 +2022/06/22-11:41:09.797780 7fdb73c14700 [tions/cf_options.cc:941] level0_slowdown_writes_trigger: 20 +2022/06/22-11:41:09.797782 7fdb73c14700 [tions/cf_options.cc:943] level0_stop_writes_trigger: 36 +2022/06/22-11:41:09.797784 7fdb73c14700 [tions/cf_options.cc:945] max_compaction_bytes: 1677721600 +2022/06/22-11:41:09.797786 7fdb73c14700 [tions/cf_options.cc:947] target_file_size_base: 67108864 +2022/06/22-11:41:09.797787 7fdb73c14700 [tions/cf_options.cc:949] target_file_size_multiplier: 1 +2022/06/22-11:41:09.797789 7fdb73c14700 [tions/cf_options.cc:951] max_bytes_for_level_base: 268435456 +2022/06/22-11:41:09.797791 7fdb73c14700 [tions/cf_options.cc:953] max_bytes_for_level_multiplier: 10.000000 +2022/06/22-11:41:09.797795 7fdb73c14700 [tions/cf_options.cc:955] ttl: 2592000 +2022/06/22-11:41:09.797797 7fdb73c14700 [tions/cf_options.cc:957] periodic_compaction_seconds: 0 +2022/06/22-11:41:09.797801 7fdb73c14700 [tions/cf_options.cc:971] max_bytes_for_level_multiplier_additional: 1, 1, 1, 1, 1, 1, 1 +2022/06/22-11:41:09.797803 7fdb73c14700 [tions/cf_options.cc:973] max_sequential_skip_in_iterations: 8 +2022/06/22-11:41:09.797805 7fdb73c14700 [tions/cf_options.cc:975] check_flush_compaction_key_order: 1 +2022/06/22-11:41:09.797807 7fdb73c14700 [tions/cf_options.cc:977] paranoid_file_checks: 0 +2022/06/22-11:41:09.797809 7fdb73c14700 [tions/cf_options.cc:979] report_bg_io_stats: 0 +2022/06/22-11:41:09.797811 7fdb73c14700 [tions/cf_options.cc:981] compression: 1 +2022/06/22-11:41:09.797813 7fdb73c14700 [tions/cf_options.cc:985] compaction_options_universal.size_ratio : 1 +2022/06/22-11:41:09.797814 7fdb73c14700 [tions/cf_options.cc:987] compaction_options_universal.min_merge_width : 2 +2022/06/22-11:41:09.797816 7fdb73c14700 [tions/cf_options.cc:989] compaction_options_universal.max_merge_width : -1 +2022/06/22-11:41:09.797818 7fdb73c14700 [tions/cf_options.cc:992] compaction_options_universal.max_size_amplification_percent : 200 +2022/06/22-11:41:09.797820 7fdb73c14700 [tions/cf_options.cc:995] compaction_options_universal.compression_size_percent : -1 +2022/06/22-11:41:09.797840 7fdb73c14700 [tions/cf_options.cc:997] compaction_options_universal.stop_style : 1 +2022/06/22-11:41:09.797842 7fdb73c14700 [tions/cf_options.cc:1000] compaction_options_universal.allow_trivial_move : 0 +2022/06/22-11:41:09.797844 7fdb73c14700 [tions/cf_options.cc:1004] compaction_options_fifo.max_table_files_size : 1073741824 +2022/06/22-11:41:09.797846 7fdb73c14700 [tions/cf_options.cc:1006] compaction_options_fifo.allow_compaction : 0 +2022/06/22-11:41:09.797848 7fdb73c14700 [tions/cf_options.cc:1010] enable_blob_files: false +2022/06/22-11:41:09.797850 7fdb73c14700 [tions/cf_options.cc:1012] min_blob_size: 0 +2022/06/22-11:41:09.797852 7fdb73c14700 [tions/cf_options.cc:1014] blob_file_size: 268435456 +2022/06/22-11:41:09.797854 7fdb73c14700 [tions/cf_options.cc:1016] blob_compression_type: NoCompression +2022/06/22-11:41:09.797856 7fdb73c14700 [tions/cf_options.cc:1018] enable_blob_garbage_collection: false +2022/06/22-11:41:09.797858 7fdb73c14700 [tions/cf_options.cc:1020] blob_garbage_collection_age_cutoff: 0.250000 +2022/06/22-11:41:09.876696 7fdb73c14700 [/db_impl/db_impl.cc:447] Shutdown: canceling all background work +2022/06/22-11:41:09.877075 7fdb73c14700 [/db_impl/db_impl.cc:625] Shutdown complete diff --git a/metadata/LOG.old.1655869269746608 b/metadata/LOG.old.1655869269746608 new file mode 100644 index 0000000000000..f98750e15f435 --- /dev/null +++ b/metadata/LOG.old.1655869269746608 @@ -0,0 +1,326 @@ +2022/06/22-11:39:41.505695 7f512435e700 RocksDB version: 6.16.4 +2022/06/22-11:39:41.505756 7f512435e700 Git sha rocksdb_build_git_sha: +2022/06/22-11:39:41.505760 7f512435e700 Compile date Mar 31 2021 +2022/06/22-11:39:41.505834 7f512435e700 DB SUMMARY +2022/06/22-11:39:41.505837 7f512435e700 DB Session ID: FGL6KF7BET3X6FEYKRF3 +2022/06/22-11:39:41.505870 7f512435e700 SST files in /home/lan/repo/git/pulsar/metadata dir, Total Num: 0, files: +2022/06/22-11:39:41.505873 7f512435e700 Write Ahead Log file in /home/lan/repo/git/pulsar/metadata: +2022/06/22-11:39:41.505877 7f512435e700 Options.error_if_exists: 0 +2022/06/22-11:39:41.505879 7f512435e700 Options.create_if_missing: 1 +2022/06/22-11:39:41.505884 7f512435e700 Options.paranoid_checks: 1 +2022/06/22-11:39:41.505886 7f512435e700 Options.track_and_verify_wals_in_manifest: 0 +2022/06/22-11:39:41.505888 7f512435e700 Options.env: 0x7f509ffbbc60 +2022/06/22-11:39:41.505891 7f512435e700 Options.fs: Posix File System +2022/06/22-11:39:41.505893 7f512435e700 Options.info_log: 0x7f5121194130 +2022/06/22-11:39:41.505895 7f512435e700 Options.max_file_opening_threads: 16 +2022/06/22-11:39:41.505898 7f512435e700 Options.statistics: (nil) +2022/06/22-11:39:41.505900 7f512435e700 Options.use_fsync: 0 +2022/06/22-11:39:41.505902 7f512435e700 Options.max_log_file_size: 0 +2022/06/22-11:39:41.505904 7f512435e700 Options.max_manifest_file_size: 1073741824 +2022/06/22-11:39:41.505906 7f512435e700 Options.log_file_time_to_roll: 86400 +2022/06/22-11:39:41.505908 7f512435e700 Options.keep_log_file_num: 30 +2022/06/22-11:39:41.505910 7f512435e700 Options.recycle_log_file_num: 0 +2022/06/22-11:39:41.505913 7f512435e700 Options.allow_fallocate: 1 +2022/06/22-11:39:41.505915 7f512435e700 Options.allow_mmap_reads: 0 +2022/06/22-11:39:41.505917 7f512435e700 Options.allow_mmap_writes: 0 +2022/06/22-11:39:41.505919 7f512435e700 Options.use_direct_reads: 0 +2022/06/22-11:39:41.505921 7f512435e700 Options.use_direct_io_for_flush_and_compaction: 0 +2022/06/22-11:39:41.505923 7f512435e700 Options.create_missing_column_families: 0 +2022/06/22-11:39:41.505925 7f512435e700 Options.db_log_dir: +2022/06/22-11:39:41.505927 7f512435e700 Options.wal_dir: /home/lan/repo/git/pulsar/metadata +2022/06/22-11:39:41.505929 7f512435e700 Options.table_cache_numshardbits: 6 +2022/06/22-11:39:41.505931 7f512435e700 Options.WAL_ttl_seconds: 0 +2022/06/22-11:39:41.505933 7f512435e700 Options.WAL_size_limit_MB: 0 +2022/06/22-11:39:41.505935 7f512435e700 Options.max_write_batch_group_size_bytes: 1048576 +2022/06/22-11:39:41.505937 7f512435e700 Options.manifest_preallocation_size: 4194304 +2022/06/22-11:39:41.505940 7f512435e700 Options.is_fd_close_on_exec: 1 +2022/06/22-11:39:41.505942 7f512435e700 Options.advise_random_on_open: 1 +2022/06/22-11:39:41.505944 7f512435e700 Options.db_write_buffer_size: 0 +2022/06/22-11:39:41.505946 7f512435e700 Options.write_buffer_manager: 0x7f5121194650 +2022/06/22-11:39:41.505948 7f512435e700 Options.access_hint_on_compaction_start: 1 +2022/06/22-11:39:41.505950 7f512435e700 Options.new_table_reader_for_compaction_inputs: 0 +2022/06/22-11:39:41.505952 7f512435e700 Options.random_access_max_buffer_size: 1048576 +2022/06/22-11:39:41.505954 7f512435e700 Options.use_adaptive_mutex: 0 +2022/06/22-11:39:41.505956 7f512435e700 Options.rate_limiter: (nil) +2022/06/22-11:39:41.505960 7f512435e700 Options.sst_file_manager.rate_bytes_per_sec: 0 +2022/06/22-11:39:41.505977 7f512435e700 Options.wal_recovery_mode: 2 +2022/06/22-11:39:41.505979 7f512435e700 Options.enable_thread_tracking: 0 +2022/06/22-11:39:41.505981 7f512435e700 Options.enable_pipelined_write: 0 +2022/06/22-11:39:41.505983 7f512435e700 Options.unordered_write: 0 +2022/06/22-11:39:41.505985 7f512435e700 Options.allow_concurrent_memtable_write: 1 +2022/06/22-11:39:41.505987 7f512435e700 Options.enable_write_thread_adaptive_yield: 1 +2022/06/22-11:39:41.505989 7f512435e700 Options.write_thread_max_yield_usec: 100 +2022/06/22-11:39:41.505991 7f512435e700 Options.write_thread_slow_yield_usec: 3 +2022/06/22-11:39:41.505993 7f512435e700 Options.row_cache: None +2022/06/22-11:39:41.505995 7f512435e700 Options.wal_filter: None +2022/06/22-11:39:41.505998 7f512435e700 Options.avoid_flush_during_recovery: 0 +2022/06/22-11:39:41.506000 7f512435e700 Options.allow_ingest_behind: 0 +2022/06/22-11:39:41.506002 7f512435e700 Options.preserve_deletes: 0 +2022/06/22-11:39:41.506004 7f512435e700 Options.two_write_queues: 0 +2022/06/22-11:39:41.506006 7f512435e700 Options.manual_wal_flush: 0 +2022/06/22-11:39:41.506008 7f512435e700 Options.atomic_flush: 0 +2022/06/22-11:39:41.506010 7f512435e700 Options.avoid_unnecessary_blocking_io: 0 +2022/06/22-11:39:41.506013 7f512435e700 Options.persist_stats_to_disk: 0 +2022/06/22-11:39:41.506015 7f512435e700 Options.write_dbid_to_manifest: 0 +2022/06/22-11:39:41.506017 7f512435e700 Options.log_readahead_size: 0 +2022/06/22-11:39:41.506019 7f512435e700 Options.file_checksum_gen_factory: Unknown +2022/06/22-11:39:41.506021 7f512435e700 Options.best_efforts_recovery: 0 +2022/06/22-11:39:41.506024 7f512435e700 Options.max_bgerror_resume_count: 2147483647 +2022/06/22-11:39:41.506026 7f512435e700 Options.bgerror_resume_retry_interval: 1000000 +2022/06/22-11:39:41.506028 7f512435e700 Options.allow_data_in_errors: 0 +2022/06/22-11:39:41.506030 7f512435e700 Options.db_host_id: __hostname__ +2022/06/22-11:39:41.506034 7f512435e700 Options.max_background_jobs: 2 +2022/06/22-11:39:41.506037 7f512435e700 Options.max_background_compactions: -1 +2022/06/22-11:39:41.506042 7f512435e700 Options.max_subcompactions: 1 +2022/06/22-11:39:41.506044 7f512435e700 Options.avoid_flush_during_shutdown: 0 +2022/06/22-11:39:41.506047 7f512435e700 Options.writable_file_max_buffer_size: 1048576 +2022/06/22-11:39:41.506049 7f512435e700 Options.delayed_write_rate : 16777216 +2022/06/22-11:39:41.506051 7f512435e700 Options.max_total_wal_size: 0 +2022/06/22-11:39:41.506053 7f512435e700 Options.delete_obsolete_files_period_micros: 21600000000 +2022/06/22-11:39:41.506055 7f512435e700 Options.stats_dump_period_sec: 600 +2022/06/22-11:39:41.506058 7f512435e700 Options.stats_persist_period_sec: 600 +2022/06/22-11:39:41.506060 7f512435e700 Options.stats_history_buffer_size: 1048576 +2022/06/22-11:39:41.506062 7f512435e700 Options.max_open_files: -1 +2022/06/22-11:39:41.506065 7f512435e700 Options.bytes_per_sync: 0 +2022/06/22-11:39:41.506067 7f512435e700 Options.wal_bytes_per_sync: 0 +2022/06/22-11:39:41.506069 7f512435e700 Options.strict_bytes_per_sync: 0 +2022/06/22-11:39:41.506071 7f512435e700 Options.compaction_readahead_size: 0 +2022/06/22-11:39:41.506074 7f512435e700 Options.max_background_flushes: -1 +2022/06/22-11:39:41.506076 7f512435e700 Compression algorithms supported: +2022/06/22-11:39:41.506079 7f512435e700 kZSTDNotFinalCompression supported: 1 +2022/06/22-11:39:41.506088 7f512435e700 kZSTD supported: 1 +2022/06/22-11:39:41.506102 7f512435e700 kXpressCompression supported: 0 +2022/06/22-11:39:41.506105 7f512435e700 kLZ4HCCompression supported: 1 +2022/06/22-11:39:41.506107 7f512435e700 kLZ4Compression supported: 1 +2022/06/22-11:39:41.506109 7f512435e700 kBZip2Compression supported: 1 +2022/06/22-11:39:41.506111 7f512435e700 kZlibCompression supported: 1 +2022/06/22-11:39:41.506114 7f512435e700 kSnappyCompression supported: 1 +2022/06/22-11:39:41.506121 7f512435e700 Fast CRC32 supported: Not supported on x86 +2022/06/22-11:39:41.515586 7f512435e700 [/db_impl/db_impl_open.cc:281] Creating manifest 1 +2022/06/22-11:39:41.529961 7f512435e700 [/version_set.cc:4757] Recovering from manifest file: /home/lan/repo/git/pulsar/metadata/MANIFEST-000001 +2022/06/22-11:39:41.530179 7f512435e700 [/column_family.cc:595] --------------- Options for column family [default]: +2022/06/22-11:39:41.530187 7f512435e700 Options.comparator: leveldb.BytewiseComparator +2022/06/22-11:39:41.530190 7f512435e700 Options.merge_operator: None +2022/06/22-11:39:41.530191 7f512435e700 Options.compaction_filter: None +2022/06/22-11:39:41.530193 7f512435e700 Options.compaction_filter_factory: None +2022/06/22-11:39:41.530195 7f512435e700 Options.sst_partitioner_factory: None +2022/06/22-11:39:41.530197 7f512435e700 Options.memtable_factory: SkipListFactory +2022/06/22-11:39:41.530199 7f512435e700 Options.table_factory: BlockBasedTable +2022/06/22-11:39:41.530239 7f512435e700 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f5121191810) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 1 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + data_block_index_type: 0 + index_shortening: 1 + data_block_hash_table_util_ratio: 0.750000 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x7f51211a5430 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: (nil) + persistent_cache: (nil) + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 4 + enable_index_compression: 1 + block_align: 0 +2022/06/22-11:39:41.530243 7f512435e700 Options.write_buffer_size: 67108864 +2022/06/22-11:39:41.530245 7f512435e700 Options.max_write_buffer_number: 2 +2022/06/22-11:39:41.530247 7f512435e700 Options.compression: Snappy +2022/06/22-11:39:41.530250 7f512435e700 Options.bottommost_compression: Disabled +2022/06/22-11:39:41.530251 7f512435e700 Options.prefix_extractor: nullptr +2022/06/22-11:39:41.530253 7f512435e700 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2022/06/22-11:39:41.530255 7f512435e700 Options.num_levels: 7 +2022/06/22-11:39:41.530257 7f512435e700 Options.min_write_buffer_number_to_merge: 1 +2022/06/22-11:39:41.530259 7f512435e700 Options.max_write_buffer_number_to_maintain: 0 +2022/06/22-11:39:41.530261 7f512435e700 Options.max_write_buffer_size_to_maintain: 134217728 +2022/06/22-11:39:41.530264 7f512435e700 Options.bottommost_compression_opts.window_bits: -14 +2022/06/22-11:39:41.530265 7f512435e700 Options.bottommost_compression_opts.level: 32767 +2022/06/22-11:39:41.530268 7f512435e700 Options.bottommost_compression_opts.strategy: 0 +2022/06/22-11:39:41.530270 7f512435e700 Options.bottommost_compression_opts.max_dict_bytes: 0 +2022/06/22-11:39:41.530272 7f512435e700 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2022/06/22-11:39:41.530274 7f512435e700 Options.bottommost_compression_opts.parallel_threads: 1 +2022/06/22-11:39:41.530296 7f512435e700 Options.bottommost_compression_opts.enabled: false +2022/06/22-11:39:41.530299 7f512435e700 Options.compression_opts.window_bits: -14 +2022/06/22-11:39:41.530301 7f512435e700 Options.compression_opts.level: 32767 +2022/06/22-11:39:41.530303 7f512435e700 Options.compression_opts.strategy: 0 +2022/06/22-11:39:41.530305 7f512435e700 Options.compression_opts.max_dict_bytes: 0 +2022/06/22-11:39:41.530307 7f512435e700 Options.compression_opts.zstd_max_train_bytes: 0 +2022/06/22-11:39:41.530309 7f512435e700 Options.compression_opts.parallel_threads: 1 +2022/06/22-11:39:41.530313 7f512435e700 Options.compression_opts.enabled: false +2022/06/22-11:39:41.530315 7f512435e700 Options.level0_file_num_compaction_trigger: 4 +2022/06/22-11:39:41.530317 7f512435e700 Options.level0_slowdown_writes_trigger: 20 +2022/06/22-11:39:41.530319 7f512435e700 Options.level0_stop_writes_trigger: 36 +2022/06/22-11:39:41.530321 7f512435e700 Options.target_file_size_base: 67108864 +2022/06/22-11:39:41.530324 7f512435e700 Options.target_file_size_multiplier: 1 +2022/06/22-11:39:41.530326 7f512435e700 Options.max_bytes_for_level_base: 268435456 +2022/06/22-11:39:41.530328 7f512435e700 Options.level_compaction_dynamic_level_bytes: 0 +2022/06/22-11:39:41.530331 7f512435e700 Options.max_bytes_for_level_multiplier: 10.000000 +2022/06/22-11:39:41.530334 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2022/06/22-11:39:41.530336 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2022/06/22-11:39:41.530338 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2022/06/22-11:39:41.530340 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2022/06/22-11:39:41.530342 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2022/06/22-11:39:41.530344 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2022/06/22-11:39:41.530346 7f512435e700 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2022/06/22-11:39:41.530348 7f512435e700 Options.max_sequential_skip_in_iterations: 8 +2022/06/22-11:39:41.530351 7f512435e700 Options.max_compaction_bytes: 1677721600 +2022/06/22-11:39:41.530353 7f512435e700 Options.arena_block_size: 8388608 +2022/06/22-11:39:41.530355 7f512435e700 Options.soft_pending_compaction_bytes_limit: 68719476736 +2022/06/22-11:39:41.530357 7f512435e700 Options.hard_pending_compaction_bytes_limit: 274877906944 +2022/06/22-11:39:41.530359 7f512435e700 Options.rate_limit_delay_max_milliseconds: 100 +2022/06/22-11:39:41.530361 7f512435e700 Options.disable_auto_compactions: 1 +2022/06/22-11:39:41.530364 7f512435e700 Options.compaction_style: kCompactionStyleLevel +2022/06/22-11:39:41.530366 7f512435e700 Options.compaction_pri: kMinOverlappingRatio +2022/06/22-11:39:41.530368 7f512435e700 Options.compaction_options_universal.size_ratio: 1 +2022/06/22-11:39:41.530370 7f512435e700 Options.compaction_options_universal.min_merge_width: 2 +2022/06/22-11:39:41.530372 7f512435e700 Options.compaction_options_universal.max_merge_width: 4294967295 +2022/06/22-11:39:41.530374 7f512435e700 Options.compaction_options_universal.max_size_amplification_percent: 200 +2022/06/22-11:39:41.530376 7f512435e700 Options.compaction_options_universal.compression_size_percent: -1 +2022/06/22-11:39:41.530378 7f512435e700 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2022/06/22-11:39:41.530380 7f512435e700 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2022/06/22-11:39:41.530382 7f512435e700 Options.compaction_options_fifo.allow_compaction: 0 +2022/06/22-11:39:41.530389 7f512435e700 Options.table_properties_collectors: +2022/06/22-11:39:41.530391 7f512435e700 Options.inplace_update_support: 0 +2022/06/22-11:39:41.530393 7f512435e700 Options.inplace_update_num_locks: 10000 +2022/06/22-11:39:41.530405 7f512435e700 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2022/06/22-11:39:41.530408 7f512435e700 Options.memtable_whole_key_filtering: 0 +2022/06/22-11:39:41.530410 7f512435e700 Options.memtable_huge_page_size: 0 +2022/06/22-11:39:41.530412 7f512435e700 Options.bloom_locality: 0 +2022/06/22-11:39:41.530413 7f512435e700 Options.max_successive_merges: 0 +2022/06/22-11:39:41.530415 7f512435e700 Options.optimize_filters_for_hits: 0 +2022/06/22-11:39:41.530417 7f512435e700 Options.paranoid_file_checks: 0 +2022/06/22-11:39:41.530419 7f512435e700 Options.force_consistency_checks: 1 +2022/06/22-11:39:41.530421 7f512435e700 Options.report_bg_io_stats: 0 +2022/06/22-11:39:41.530423 7f512435e700 Options.ttl: 2592000 +2022/06/22-11:39:41.530425 7f512435e700 Options.periodic_compaction_seconds: 0 +2022/06/22-11:39:41.530427 7f512435e700 Options.enable_blob_files: false +2022/06/22-11:39:41.530429 7f512435e700 Options.min_blob_size: 0 +2022/06/22-11:39:41.530431 7f512435e700 Options.blob_file_size: 268435456 +2022/06/22-11:39:41.530433 7f512435e700 Options.blob_compression_type: NoCompression +2022/06/22-11:39:41.530435 7f512435e700 Options.enable_blob_garbage_collection: false +2022/06/22-11:39:41.530438 7f512435e700 Options.blob_garbage_collection_age_cutoff: 0.250000 +2022/06/22-11:39:41.532186 7f512435e700 [/version_set.cc:4805] Recovered from manifest file:/home/lan/repo/git/pulsar/metadata/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2022/06/22-11:39:41.532200 7f512435e700 [/version_set.cc:4814] Column family [default] (ID 0), log number is 0 +2022/06/22-11:39:41.532345 7f512435e700 [/version_set.cc:4111] Creating manifest 3 +2022/06/22-11:39:41.551101 7f512435e700 [/db_impl/db_impl_open.cc:1705] SstFileManager instance 0x7f5120ff9910 +2022/06/22-11:39:41.551128 7f512435e700 DB pointer 0x7f5121196ea0 +2022/06/22-11:39:41.551193 7f512435e700 [WARN] [ilities/transactions/pessimistic_transaction_db.cc:244] Transaction write_policy is 0 +2022/06/22-11:39:41.552901 7f5086ff5700 [/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- +2022/06/22-11:39:41.552942 7f5086ff5700 [/db_impl/db_impl.cc:903] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count + +** File Read Latency Histogram By Level [default] ** +2022/06/22-11:39:41.560029 7f512435e700 [/db_impl/db_impl.cc:1006] SetOptions() on column family [default], inputs: +2022/06/22-11:39:41.560039 7f512435e700 [/db_impl/db_impl.cc:1009] disable_auto_compactions: false +2022/06/22-11:39:41.560041 7f512435e700 [/db_impl/db_impl.cc:1013] [default] SetOptions() succeeded +2022/06/22-11:39:41.560045 7f512435e700 [tions/cf_options.cc:910] write_buffer_size: 67108864 +2022/06/22-11:39:41.560047 7f512435e700 [tions/cf_options.cc:912] max_write_buffer_number: 2 +2022/06/22-11:39:41.560049 7f512435e700 [tions/cf_options.cc:915] arena_block_size: 8388608 +2022/06/22-11:39:41.560051 7f512435e700 [tions/cf_options.cc:917] memtable_prefix_bloom_ratio: 0.000000 +2022/06/22-11:39:41.560053 7f512435e700 [tions/cf_options.cc:919] memtable_whole_key_filtering: 0 +2022/06/22-11:39:41.560055 7f512435e700 [tions/cf_options.cc:922] memtable_huge_page_size: 0 +2022/06/22-11:39:41.560057 7f512435e700 [tions/cf_options.cc:925] max_successive_merges: 0 +2022/06/22-11:39:41.560058 7f512435e700 [tions/cf_options.cc:928] inplace_update_num_locks: 10000 +2022/06/22-11:39:41.560060 7f512435e700 [tions/cf_options.cc:931] prefix_extractor: nullptr +2022/06/22-11:39:41.560062 7f512435e700 [tions/cf_options.cc:933] disable_auto_compactions: 0 +2022/06/22-11:39:41.560063 7f512435e700 [tions/cf_options.cc:935] soft_pending_compaction_bytes_limit: 68719476736 +2022/06/22-11:39:41.560065 7f512435e700 [tions/cf_options.cc:937] hard_pending_compaction_bytes_limit: 274877906944 +2022/06/22-11:39:41.560067 7f512435e700 [tions/cf_options.cc:939] level0_file_num_compaction_trigger: 4 +2022/06/22-11:39:41.560068 7f512435e700 [tions/cf_options.cc:941] level0_slowdown_writes_trigger: 20 +2022/06/22-11:39:41.560070 7f512435e700 [tions/cf_options.cc:943] level0_stop_writes_trigger: 36 +2022/06/22-11:39:41.560072 7f512435e700 [tions/cf_options.cc:945] max_compaction_bytes: 1677721600 +2022/06/22-11:39:41.560073 7f512435e700 [tions/cf_options.cc:947] target_file_size_base: 67108864 +2022/06/22-11:39:41.560075 7f512435e700 [tions/cf_options.cc:949] target_file_size_multiplier: 1 +2022/06/22-11:39:41.560076 7f512435e700 [tions/cf_options.cc:951] max_bytes_for_level_base: 268435456 +2022/06/22-11:39:41.560078 7f512435e700 [tions/cf_options.cc:953] max_bytes_for_level_multiplier: 10.000000 +2022/06/22-11:39:41.560081 7f512435e700 [tions/cf_options.cc:955] ttl: 2592000 +2022/06/22-11:39:41.560083 7f512435e700 [tions/cf_options.cc:957] periodic_compaction_seconds: 0 +2022/06/22-11:39:41.560086 7f512435e700 [tions/cf_options.cc:971] max_bytes_for_level_multiplier_additional: 1, 1, 1, 1, 1, 1, 1 +2022/06/22-11:39:41.560088 7f512435e700 [tions/cf_options.cc:973] max_sequential_skip_in_iterations: 8 +2022/06/22-11:39:41.560090 7f512435e700 [tions/cf_options.cc:975] check_flush_compaction_key_order: 1 +2022/06/22-11:39:41.560092 7f512435e700 [tions/cf_options.cc:977] paranoid_file_checks: 0 +2022/06/22-11:39:41.560093 7f512435e700 [tions/cf_options.cc:979] report_bg_io_stats: 0 +2022/06/22-11:39:41.560095 7f512435e700 [tions/cf_options.cc:981] compression: 1 +2022/06/22-11:39:41.560097 7f512435e700 [tions/cf_options.cc:985] compaction_options_universal.size_ratio : 1 +2022/06/22-11:39:41.560098 7f512435e700 [tions/cf_options.cc:987] compaction_options_universal.min_merge_width : 2 +2022/06/22-11:39:41.560100 7f512435e700 [tions/cf_options.cc:989] compaction_options_universal.max_merge_width : -1 +2022/06/22-11:39:41.560101 7f512435e700 [tions/cf_options.cc:992] compaction_options_universal.max_size_amplification_percent : 200 +2022/06/22-11:39:41.560103 7f512435e700 [tions/cf_options.cc:995] compaction_options_universal.compression_size_percent : -1 +2022/06/22-11:39:41.560124 7f512435e700 [tions/cf_options.cc:997] compaction_options_universal.stop_style : 1 +2022/06/22-11:39:41.560127 7f512435e700 [tions/cf_options.cc:1000] compaction_options_universal.allow_trivial_move : 0 +2022/06/22-11:39:41.560129 7f512435e700 [tions/cf_options.cc:1004] compaction_options_fifo.max_table_files_size : 1073741824 +2022/06/22-11:39:41.560130 7f512435e700 [tions/cf_options.cc:1006] compaction_options_fifo.allow_compaction : 0 +2022/06/22-11:39:41.560132 7f512435e700 [tions/cf_options.cc:1010] enable_blob_files: false +2022/06/22-11:39:41.560133 7f512435e700 [tions/cf_options.cc:1012] min_blob_size: 0 +2022/06/22-11:39:41.560135 7f512435e700 [tions/cf_options.cc:1014] blob_file_size: 268435456 +2022/06/22-11:39:41.560137 7f512435e700 [tions/cf_options.cc:1016] blob_compression_type: NoCompression +2022/06/22-11:39:41.560139 7f512435e700 [tions/cf_options.cc:1018] enable_blob_garbage_collection: false +2022/06/22-11:39:41.560141 7f512435e700 [tions/cf_options.cc:1020] blob_garbage_collection_age_cutoff: 0.250000 +2022/06/22-11:39:41.651607 7f512435e700 [/db_impl/db_impl.cc:447] Shutdown: canceling all background work +2022/06/22-11:39:41.651986 7f512435e700 [/db_impl/db_impl.cc:625] Shutdown complete diff --git a/metadata/MANIFEST-000010 b/metadata/MANIFEST-000010 new file mode 100644 index 0000000000000..92284734ef518 Binary files /dev/null and b/metadata/MANIFEST-000010 differ diff --git a/metadata/OPTIONS-000013 b/metadata/OPTIONS-000013 new file mode 100644 index 0000000000000..a48c426f96ca4 --- /dev/null +++ b/metadata/OPTIONS-000013 @@ -0,0 +1,179 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.16.4 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + wal_bytes_per_sync=0 + bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + stats_dump_period_sec=600 + stats_persist_period_sec=600 + delete_obsolete_files_period_micros=21600000000 + max_total_wal_size=0 + strict_bytes_per_sync=false + delayed_write_rate=16777216 + avoid_flush_during_shutdown=false + writable_file_max_buffer_size=1048576 + max_subcompactions=1 + base_background_compactions=-1 + max_background_compactions=-1 + max_background_jobs=2 + allow_data_in_errors=false + avoid_unnecessary_blocking_io=false + log_readahead_size=0 + manual_wal_flush=false + two_write_queues=false + write_dbid_to_manifest=false + allow_ingest_behind=false + avoid_flush_during_recovery=false + best_efforts_recovery=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + write_thread_max_yield_usec=100 + max_write_batch_group_size_bytes=1048576 + write_thread_slow_yield_usec=3 + wal_recovery_mode=kPointInTimeRecovery + allow_concurrent_memtable_write=true + db_host_id=__hostname__ + unordered_write=false + enable_pipelined_write=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_size_limit_MB=0 + bgerror_resume_retry_interval=1000000 + wal_dir=/home/lan/repo/git/pulsar/metadata + log_file_time_to_roll=86400 + keep_log_file_num=30 + WAL_ttl_seconds=0 + db_write_buffer_size=0 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + max_bgerror_resume_count=2147483647 + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + track_and_verify_wals_in_manifest=false + dump_malloc_stats=false + paranoid_checks=true + is_fd_close_on_exec=true + max_manifest_file_size=1073741824 + error_if_exists=false + use_adaptive_mutex=false + atomic_flush=false + enable_thread_tracking=false + create_missing_column_families=false + create_if_missing=true + manifest_preallocation_size=4194304 + use_fsync=false + allow_2pc=true + recycle_log_file_num=0 + use_direct_io_for_flush_and_compaction=false + use_direct_reads=false + allow_mmap_writes=false + preserve_deletes=false + enable_write_thread_adaptive_yield=true + max_log_file_size=0 + allow_fallocate=true + allow_mmap_reads=false + new_table_reader_for_compaction_inputs=false + advise_random_on_open=true + + +[CFOptions "default"] + bottommost_compression=kDisableCompressionOption + bottommost_compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + sample_for_compression=0 + blob_file_size=268435456 + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;} + compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} + prefix_extractor=nullptr + max_bytes_for_level_multiplier_additional=1:{1}:{1}:{1}:{1}:{1}:{1} + max_bytes_for_level_base=268435456 + memtable_whole_key_filtering=false + memtable_prefix_bloom_size_ratio=0.000000 + enable_blob_files=false + target_file_size_base=67108864 + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + max_sequential_skip_in_iterations=8 + arena_block_size=8388608 + target_file_size_multiplier=1 + max_write_buffer_number=2 + write_buffer_size=67108864 + blob_compression_type=kNoCompression + compression=kSnappyCompression + level0_stop_writes_trigger=36 + level0_slowdown_writes_trigger=20 + level0_file_num_compaction_trigger=4 + ttl=2592000 + max_compaction_bytes=1677721600 + blob_garbage_collection_age_cutoff=0.250000 + compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + enable_blob_garbage_collection=false + soft_pending_compaction_bytes_limit=68719476736 + paranoid_file_checks=false + periodic_compaction_seconds=0 + check_flush_compaction_key_order=true + min_blob_size=0 + hard_pending_compaction_bytes_limit=274877906944 + disable_auto_compactions=true + max_bytes_for_level_multiplier=10.000000 + report_bg_io_stats=false + compaction_pri=kMinOverlappingRatio + compaction_filter_factory=nullptr + comparator=leveldb.BytewiseComparator + bloom_locality=0 + compaction_style=kCompactionStyleLevel + min_write_buffer_number_to_merge=1 + max_write_buffer_size_to_maintain=134217728 + max_write_buffer_number_to_maintain=0 + merge_operator=nullptr + memtable_factory=SkipListFactory + memtable_insert_with_hint_prefix_extractor=nullptr + num_levels=7 + force_consistency_checks=true + optimize_filters_for_hits=false + compaction_filter=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + table_factory=BlockBasedTable + +[TableOptions/BlockBasedTable "default"] + pin_top_level_index_and_filter=true + block_align=false + read_amp_bytes_per_bit=0 + verify_compression=false + enable_index_compression=true + whole_key_filtering=true + optimize_filters_for_memory=false + index_block_restart_interval=1 + block_restart_interval=16 + block_size=4096 + format_version=4 + partition_filters=false + block_size_deviation=10 + no_block_cache=false + checksum=kCRC32c + data_block_hash_table_util_ratio=0.750000 + index_shortening=kShortenSeparators + data_block_index_type=kDataBlockBinarySearch + hash_index_allow_collision=true + filter_policy=nullptr + metadata_block_size=4096 + index_type=kBinarySearch + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/metadata/OPTIONS-000015 b/metadata/OPTIONS-000015 new file mode 100644 index 0000000000000..7da11a71938b2 --- /dev/null +++ b/metadata/OPTIONS-000015 @@ -0,0 +1,179 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.16.4 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + wal_bytes_per_sync=0 + bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + stats_dump_period_sec=600 + stats_persist_period_sec=600 + delete_obsolete_files_period_micros=21600000000 + max_total_wal_size=0 + strict_bytes_per_sync=false + delayed_write_rate=16777216 + avoid_flush_during_shutdown=false + writable_file_max_buffer_size=1048576 + max_subcompactions=1 + base_background_compactions=-1 + max_background_compactions=-1 + max_background_jobs=2 + allow_data_in_errors=false + avoid_unnecessary_blocking_io=false + log_readahead_size=0 + manual_wal_flush=false + two_write_queues=false + write_dbid_to_manifest=false + allow_ingest_behind=false + avoid_flush_during_recovery=false + best_efforts_recovery=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + write_thread_max_yield_usec=100 + max_write_batch_group_size_bytes=1048576 + write_thread_slow_yield_usec=3 + wal_recovery_mode=kPointInTimeRecovery + allow_concurrent_memtable_write=true + db_host_id=__hostname__ + unordered_write=false + enable_pipelined_write=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_size_limit_MB=0 + bgerror_resume_retry_interval=1000000 + wal_dir=/home/lan/repo/git/pulsar/metadata + log_file_time_to_roll=86400 + keep_log_file_num=30 + WAL_ttl_seconds=0 + db_write_buffer_size=0 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + max_bgerror_resume_count=2147483647 + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + track_and_verify_wals_in_manifest=false + dump_malloc_stats=false + paranoid_checks=true + is_fd_close_on_exec=true + max_manifest_file_size=1073741824 + error_if_exists=false + use_adaptive_mutex=false + atomic_flush=false + enable_thread_tracking=false + create_missing_column_families=false + create_if_missing=true + manifest_preallocation_size=4194304 + use_fsync=false + allow_2pc=true + recycle_log_file_num=0 + use_direct_io_for_flush_and_compaction=false + use_direct_reads=false + allow_mmap_writes=false + preserve_deletes=false + enable_write_thread_adaptive_yield=true + max_log_file_size=0 + allow_fallocate=true + allow_mmap_reads=false + new_table_reader_for_compaction_inputs=false + advise_random_on_open=true + + +[CFOptions "default"] + bottommost_compression=kDisableCompressionOption + bottommost_compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + sample_for_compression=0 + blob_file_size=268435456 + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;} + compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} + prefix_extractor=nullptr + max_bytes_for_level_multiplier_additional=1:{1}:{1}:{1}:{1}:{1}:{1} + max_bytes_for_level_base=268435456 + memtable_whole_key_filtering=false + memtable_prefix_bloom_size_ratio=0.000000 + enable_blob_files=false + target_file_size_base=67108864 + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + max_sequential_skip_in_iterations=8 + arena_block_size=8388608 + target_file_size_multiplier=1 + max_write_buffer_number=2 + write_buffer_size=67108864 + blob_compression_type=kNoCompression + compression=kSnappyCompression + level0_stop_writes_trigger=36 + level0_slowdown_writes_trigger=20 + level0_file_num_compaction_trigger=4 + ttl=2592000 + max_compaction_bytes=1677721600 + blob_garbage_collection_age_cutoff=0.250000 + compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + enable_blob_garbage_collection=false + soft_pending_compaction_bytes_limit=68719476736 + paranoid_file_checks=false + periodic_compaction_seconds=0 + check_flush_compaction_key_order=true + min_blob_size=0 + hard_pending_compaction_bytes_limit=274877906944 + disable_auto_compactions=false + max_bytes_for_level_multiplier=10.000000 + report_bg_io_stats=false + compaction_pri=kMinOverlappingRatio + compaction_filter_factory=nullptr + comparator=leveldb.BytewiseComparator + bloom_locality=0 + compaction_style=kCompactionStyleLevel + min_write_buffer_number_to_merge=1 + max_write_buffer_size_to_maintain=134217728 + max_write_buffer_number_to_maintain=0 + merge_operator=nullptr + memtable_factory=SkipListFactory + memtable_insert_with_hint_prefix_extractor=nullptr + num_levels=7 + force_consistency_checks=true + optimize_filters_for_hits=false + compaction_filter=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + table_factory=BlockBasedTable + +[TableOptions/BlockBasedTable "default"] + pin_top_level_index_and_filter=true + block_align=false + read_amp_bytes_per_bit=0 + verify_compression=false + enable_index_compression=true + whole_key_filtering=true + optimize_filters_for_memory=false + index_block_restart_interval=1 + block_restart_interval=16 + block_size=4096 + format_version=4 + partition_filters=false + block_size_deviation=10 + no_block_cache=false + checksum=kCRC32c + data_block_hash_table_util_ratio=0.750000 + index_shortening=kShortenSeparators + data_block_index_type=kDataBlockBinarySearch + hash_index_allow_collision=true + filter_policy=nullptr + metadata_block_size=4096 + index_type=kBinarySearch + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/pom.xml b/pom.xml index 8bd36cc63d263..ef37c60db1cf6 100644 --- a/pom.xml +++ b/pom.xml @@ -1573,7 +1573,8 @@ flexible messaging model and an intuitive client API. SCRIPT_STYLE SCRIPT_STYLE SCRIPT_STYLE - + + true @@ -2136,7 +2137,7 @@ flexible messaging model and an intuitive client API. distribution docker - tests + diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java index ee537431280f3..80574c5db6f09 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java @@ -54,7 +54,10 @@ public CompletableFuture> listPersistentTopicsAsync(NamespaceName n String path = MANAGED_LEDGER_PATH + "/" + ns + "/persistent"; return store.getChildren(path).thenApply(children -> - children.stream().map(c -> TopicName.get(TopicDomain.persistent.toString(), ns, decode(c)).toString()) + children.stream().map(c -> { + System.out.println("lan.dev.c:"+c); + return TopicName.get(TopicDomain.persistent.toString(), ns, decode(c)).toString(); + }) .collect(Collectors.toList()) ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index 4fd22eba785b9..bc29adfc69e4f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -294,6 +294,8 @@ public void start() throws Exception { if (!this.isOnlyBroker()) { if (usingNewDefaultsPIP117) { +// startBookieWithRocksDB(); +// startBookieWithMetadata("etcd://127.0.0.1:2379"); startBookieWithMetadataStore(); } else { startBookieWithZookeeper(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index c69c6e2418f7c..0d4a365fcfb7f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1711,6 +1711,9 @@ private void startWorkerService(AuthenticationService authenticationService, getInternalConfigurationData() ); + LOG.info("authenticationService:{}",authenticationService); + LOG.info("authorizationService:{}",authorizationService); + // TODO figure out how to handle errors from function worker service functionWorkerService.get().start( authenticationService, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 1e48052a6f736..2db6cff26dbfe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -231,6 +231,7 @@ private static Path getReplacedNICPath(String template, String nic) { } private static String readTrimStringFromFile(Path path) throws IOException { + System.out.println("readTrimStringFromFile:"+path); return new String(Files.readAllBytes(path), StandardCharsets.UTF_8).trim(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index a3cef4bd0276b..95979ee57a55c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1466,7 +1466,10 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, managedLedgerConfig.setProperties(properties); // Once we have the configuration, we can proceed with the async open operation - managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, + + String encodingTopic = topicName.getPersistenceNamingEncoding(); + + managedLedgerFactory.asyncOpen(encodingTopic, managedLedgerConfig, new OpenLedgerCallback() { @Override public void openLedgerComplete(ManagedLedger ledger, Object ctx) { @@ -2777,6 +2780,7 @@ private ConcurrentOpenHashMap getRuntimeConfigurationMap() { field.setAccessible(true); try { Object configValue = field.get(pulsar.getConfiguration()); + log.info("field.getName():{}|{}|{}",field.getName(),configValue == null,configValue); runtimeConfigurationMap.put(field.getName(), configValue == null ? "" : configValue); } catch (Exception e) { log.error("Failed to get value of field {}, {}", field.getName(), e.getMessage()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 49f2ac2f548cf..bbec169e51f89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1522,9 +1522,11 @@ public void checkCompaction() { ? ledger.getTotalSize() : ledger.getEstimatedBacklogSize(); } + log.info("lan.compaction:{}|{}|{}",topic,backlogEstimate,compactionThreshold); if (backlogEstimate > compactionThreshold) { try { + log.info("lan.compaction.trigger:{}|{}|{}",topic,backlogEstimate,compactionThreshold); triggerCompaction(); } catch (AlreadyRunningException are) { log.debug("[{}] Compaction already running, so don't trigger again, " diff --git a/pulsar-broker/src/main/resources/log4j2.xml b/pulsar-broker/src/main/resources/log4j2.xml new file mode 100644 index 0000000000000..985917466cfe6 --- /dev/null +++ b/pulsar-broker/src/main/resources/log4j2.xml @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableviewTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableviewTest.java new file mode 100644 index 0000000000000..0ef2a7c4adf38 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableviewTest.java @@ -0,0 +1,87 @@ +//package org.apache.pulsar.client.impl; +// +//import com.google.protobuf.ListValue; +//import com.google.protobuf.Value; +//import org.apache.commons.lang3.RandomUtils; +//import org.apache.pulsar.client.api.*; +//import org.apache.pulsar.client.api.interceptor.ProducerInterceptor; +// +//import java.util.Map; +//import java.util.concurrent.CompletableFuture; +//import java.util.concurrent.TimeUnit; +//import java.util.stream.Collectors; +// +//public class TableviewTest { +// public static void main(String[] args) throws PulsarClientException, InterruptedException { +// String topic = "persistent://public/default/testtt"; +// +// String brokerUrl = "pulsar://smartoilets.cn:26650"; +// +// PulsarClient client = PulsarClient.builder() +// .serviceUrl(brokerUrl) +// .enableTlsHostnameVerification(false) // false by default, in any case +// .allowTlsInsecureConnection(true) // false by default, in any case +// .build(); +// Producer producer = client.newProducer(Schema.STRING) +// .producerName("producer4") +// .autoUpdatePartitions(true) +// .topic(topic) +// .create(); +// +// boolean subscribeFlag = true; +// +// if(subscribeFlag){ +// Consumer subscribe = client.newConsumer(Schema.STRING) +// .enableBatchIndexAcknowledgment(true) +// .autoUpdatePartitions(true) +// .topicsPattern(topic) +// .subscriptionName("hello23") +// .subscriptionTopicsMode(RegexSubscriptionMode.AllTopics) +// .consumerName("hello") +// .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) +// +// .subscribe(); +// +// //RegexSubscriptionMode +// CompletableFuture.runAsync(()->{ +// while (true){ +// try { +// Message receive = subscribe.receive(1, TimeUnit.SECONDS); +// if(receive!=null){ +// System.out.println(""); +// } +// } catch (PulsarClientException e) { +// e.printStackTrace(); +// } +// } +// }); +// } +// TimeUnit.SECONDS.sleep(3); +// +// boolean produce = false; +// if(produce){ +// while(true){ +// TimeUnit.SECONDS.sleep(2); +// if(producer.isConnected()){ +// System.out.println("producer.send.message:"+producer.getTopic()+"|"+producer.getProducerName()); +// TypedMessageBuilder typedMessageBuilder = producer.newMessage(); +// String data = String.valueOf(RandomUtils.nextInt(10,100)); +// +// typedMessageBuilder = typedMessageBuilder +// .property("siteName","aeon") +// .property("wcName","gt3") +// .property("room","male") +// .eventTime(System.currentTimeMillis()) +// .key(data) +// .value("world"); +// typedMessageBuilder.send(); +// }else{ +// System.out.println("produce.is.not .connect!"+producer.getProducerName()); +// } +// } +// } +// +// +// } +// +//} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index 5f03c60ece906..96e656378c803 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -325,6 +325,11 @@ public String getPersistenceNamingEncoding() { if (isV2()) { return String.format("%s/%s/%s/%s", tenant, namespacePortion, domain, getEncodedLocalName()); } else { + System.out.println("tenant:"+tenant); + System.out.println("cluster:"+cluster); + System.out.println("namespacePortion:"+namespacePortion); + System.out.println("domain:"+domain); + System.out.println("getEncodedLocalName():"+getEncodedLocalName()); return String.format("%s/%s/%s/%s/%s", tenant, cluster, namespacePortion, domain, getEncodedLocalName()); } } diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go index c2ff443fcc40c..fc8a6b5b13392 100644 --- a/pulsar-function-go/conf/conf.go +++ b/pulsar-function-go/conf/conf.go @@ -77,6 +77,9 @@ type Conf struct { UserConfig string `json:"userConfig" yaml:"userConfig"` //metrics config MetricsPort int `json:"metricsPort" yaml:"metricsPort"` + + ClientAuthPlugin string `json:"ClientAuthPlugin" yaml:"ClientAuthPlugin"` + ClientAuthParams string `json:"ClientAuthParams" yaml:"ClientAuthParams"` } var ( @@ -121,6 +124,8 @@ func (c *Conf) GetConf() *Conf { return nil } } + log.Info(c.UserConfig) + log.Info(c.ClientAuthParams) return c } diff --git a/pulsar-function-go/conf/conf.yaml b/pulsar-function-go/conf/conf.yaml index 59ac9bbd51308..05a15bfa33244 100644 --- a/pulsar-function-go/conf/conf.yaml +++ b/pulsar-function-go/conf/conf.yaml @@ -59,4 +59,12 @@ maxMessageRetries: 0 deadLetterTopic: "" expectedHealthCheckInterval: 3 # metrics config -metricsPort: 50001 \ No newline at end of file +metricsPort: 50001 + + +# +clientAuthPlugin: org.apache.pulsar.client.impl.auth.AuthenticationToken +clientAuthParams: '{"token":"abc"}' + + +/home/lan/repo/git/pulsar/download/pulsar_functions/public/default/logfungo/0/logfungo -instance-conf '{"pulsarServiceURL":"pulsar://localhost:6650","instanceID":0,"funcID":"30d1b583-6f92-4b60-9bb5-66e4dcfaab72","funcVersion":"1a8a7f04-7c00-4072-8edc-d5294aec2a95","maxBufTuples":1024,"port":38179,"clusterName":"standalone","killAfterIdleMs":0,"expectedHealthCheckInterval":0,"tenant":"public","nameSpace":"default","name":"logfungo","className":"","logTopic":"","processingGuarantees":0,"secretsMap":"","userConfig":"{\"hello\":\"world\"}","runtime":3,"autoAck":true,"parallelism":1,"subscriptionType":0,"timeoutMs":0,"subscriptionName":"","cleanupSubscription":true,"subscriptionPosition":0,"sourceSpecsTopic":"test2","sourceSchemaType":"","receiverQueueSize":0,"sinkSpecsTopic":"","sinkSchemaType":"","cpu":1.0,"ram":1073741824,"disk":10737418240,"maxMessageRetries":0,"deadLetterTopic":"","metricsPort":36845,"clientAuthPlugin":"org.apache.pulsar.client.impl.auth.AuthenticationToken","clientAuthParams":"{\"token\":\"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.WkgQ1xTmRikzq-qMYlLD1wWdA0Ur0-1cO6N3mhKt8yI\"}","regexPatternSubscription":false}' \ No newline at end of file diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index 074b50d2e66e9..2b3f1b0c478ce 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -5,8 +5,6 @@ go 1.13 require ( github.com/apache/pulsar-client-go v0.8.1 github.com/apache/pulsar/pulsar-function-go v0.0.0 - github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32 // indirect - github.com/yahoo/athenz v1.8.55 // indirect ) replace github.com/apache/pulsar/pulsar-function-go => ../ diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index d167adb92fe41..0c88121d42346 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -37,18 +37,13 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4= -github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= +github.com/99designs/keyring v1.1.6 h1:kVDC2uCgVwecxCk+9zoCt2uEL6dt+dfVzMvGgnVcIuM= github.com/99designs/keyring v1.1.6/go.mod h1:16e0ds7LGQQcT59QqkTg72Hh5ShM51Byv5PEmW6uoRU= -github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I= -github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc= +github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32 h1:/gZKpgSMydtrih81nvUhlkXpZIUfthKShSCVbRzBt9Y= -github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -56,24 +51,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/pulsar-client-go v0.0.0-20200113085434-9b739cf9d098/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed h1:Lp7eU5ym84jPmIXoonoaJWVN6psyB90Olookp61LCeA= -github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.1.0/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.2.0 h1:7teu0FaXzzKPjDdUNjA7dVYKFjCy6OVX5as6nUww4qk= -github.com/apache/pulsar-client-go v0.2.0/go.mod h1:POSPPmXv1RuoM7FzHaS3NurCSOopwin2ekGK2PcOgVM= -github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825 h1:RfvcnGzo67yEHA+eDjoeAEwx5ZxWDgIoMGHJ/Z6Zq9A= -github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825/go.mod h1:pTmScVVHRhbB8wh0J+m5ZzHI0Lyfe0TwfPEbYEh+JUw= -github.com/apache/pulsar-client-go v0.7.0 h1:sZBkjJPHC7akM8n8DuzkLdwioKPSzyub3efCJ1Ltw9Y= -github.com/apache/pulsar-client-go v0.7.0/go.mod h1:EauTUv9sTmP9QRznRgK9hxnzCsIVfS8fyhTfGcuJBrE= github.com/apache/pulsar-client-go v0.8.1 h1:UZINLbH3I5YtNzqkju7g9vrl4CKrEgYSx2rbpvGufrE= github.com/apache/pulsar-client-go v0.8.1/go.mod h1:yJNcvn/IurarFDxwmoZvb2Ieylg630ifxeO/iXpk27I= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb h1:E1P0FudxDdj2RhbveZC9i3PwukLCA/4XQSkBS/dw6/I= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= +github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e h1:EqiJ0Xil8NmcXyupNqXV9oYDBeWntEIegxLahrTr8DY= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e/go.mod h1:Xee4tgYLFpYcPMcTfBYWE1uKRzeciodGTSEDMzsR6i8= -github.com/apache/pulsar/pulsar-function-go v0.0.0-20200124033432-ec122ed9562c/go.mod h1:2a3PacwSg4KPcGxO3bjH29xsoKSuSkq2mG0sjKtxsP4= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= @@ -81,7 +62,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -90,7 +70,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -104,16 +83,14 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU= github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= -github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:inRp+etsHuvVqMPNTXaFlpf/Tj7wqviBtdJoPVrPEFQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b h1:HBah4D48ypg3J7Np4N+HY/ZR76fx3HEUGxDU6Uk39oQ= github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -124,6 +101,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -137,14 +115,14 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -160,7 +138,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -171,12 +148,11 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -192,6 +168,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -209,14 +186,14 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -243,12 +220,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -258,20 +233,20 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM= github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= -github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= @@ -299,11 +274,14 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -311,7 +289,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -321,26 +298,22 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -349,9 +322,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -359,25 +331,22 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0= -github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -402,7 +371,6 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -451,7 +419,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -471,14 +438,11 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -487,12 +451,12 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -500,6 +464,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -521,7 +486,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -529,13 +493,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -551,12 +513,10 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -571,24 +531,22 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -601,7 +559,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -643,6 +600,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -672,6 +630,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -679,7 +638,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -699,7 +657,6 @@ google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -715,15 +672,14 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= @@ -737,6 +693,7 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -744,36 +701,36 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pulsar-function-go/examples/hello/hello b/pulsar-function-go/examples/hello/hello new file mode 100755 index 0000000000000..fd2511652529a Binary files /dev/null and b/pulsar-function-go/examples/hello/hello differ diff --git a/pulsar-function-go/examples/logFunc/logFunc b/pulsar-function-go/examples/logFunc/logFunc new file mode 100755 index 0000000000000..88e3409dffc06 Binary files /dev/null and b/pulsar-function-go/examples/logFunc/logFunc differ diff --git a/pulsar-function-go/gofunction b/pulsar-function-go/gofunction new file mode 100755 index 0000000000000..2c82e103d58c9 Binary files /dev/null and b/pulsar-function-go/gofunction differ diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 5e05c070e65f9..d45796e3aee04 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -189,9 +189,18 @@ CLOSE: } func (gi *goInstance) setupClient() error { - client, err := pulsar.NewClient(pulsar.ClientOptions{ - URL: gi.context.instanceConf.pulsarServiceURL, + log.Info(gi.context.instanceConf.clientAuthPlugin) + log.Info(gi.context.instanceConf.clientAuthParams) + + authentication, err2 := pulsar.NewAuthentication(gi.context.instanceConf.clientAuthPlugin, gi.context.instanceConf.clientAuthParams) + if err2 != nil { + panic(err2) + } + + client, err := pulsar.NewClient(pulsar.ClientOptions{ + Authentication: authentication, + URL: gi.context.instanceConf.pulsarServiceURL, }) if err != nil { log.Errorf("create client error:%v", err) diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go index d60beef29e8d3..4bd66342a4013 100644 --- a/pulsar-function-go/pf/instanceConf.go +++ b/pulsar-function-go/pf/instanceConf.go @@ -41,6 +41,9 @@ type instanceConf struct { killAfterIdle time.Duration expectedHealthCheckInterval int32 metricsPort int + + clientAuthPlugin string + clientAuthParams string } func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { @@ -96,6 +99,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { }, UserConfig: cfg.UserConfig, }, + clientAuthPlugin: cfg.ClientAuthPlugin, + clientAuthParams: cfg.ClientAuthParams, } if instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_EFFECTIVELY_ONCE { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java index 85e73d32a6c4a..dccfde31e5543 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java @@ -69,4 +69,7 @@ public class GoInstanceConfig { private String deadLetterTopic = ""; private int metricsPort; + + private String clientAuthPlugin; + private String clientAuthParams; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java index 2dc6a5906bf8b..af0555979845a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/windowing/triggers/TimeTriggerPolicy.java @@ -102,6 +102,7 @@ private Runnable newTriggerTask() { return new Runnable() { @Override public void run() { + log.info("lan.function.window.time.trigger:{}",context.getFunctionName()); // initialize the thread context ThreadContext.put("function", WindowUtils.getFullyQualifiedName( context.getTenant(), context.getNamespace(), context.getFunctionName())); diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index c28fda24c3f2a..64ec9629a7625 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -55,23 +55,23 @@ - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - check - verify - - check - - - - + + + + + + + + + + + + + + + + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/IncrFunction.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/IncrFunction.java new file mode 100644 index 0000000000000..724a66724f9dc --- /dev/null +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/IncrFunction.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.api.examples; + +import org.apache.pulsar.functions.api.Context; +import org.apache.pulsar.functions.api.Function; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * A function that demonstrates how to redirect logging to a topic. + * In this particular example, for every input string, the function + * does some logging. If --logTopic topic is specified, these log statements + * end up in that specified pulsar topic. + */ +public class IncrFunction implements Function { + + @Override + public String process(String input, Context context) { + Map userConfigMap = context.getUserConfigMap(); + if(userConfigMap.containsKey("incrkey")) { + String incrkey = userConfigMap.get("incrkey").toString(); + context.incrCounter(incrkey,1); + } + return String.format("%s!", input); + } + +} diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/StateFunction.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/StateFunction.java new file mode 100644 index 0000000000000..084c5cca57f7f --- /dev/null +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/StateFunction.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.api.examples; + +import org.apache.pulsar.functions.api.Context; +import org.apache.pulsar.functions.api.Function; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * A function that demonstrates how to redirect logging to a topic. + * In this particular example, for every input string, the function + * does some logging. If --logTopic topic is specified, these log statements + * end up in that specified pulsar topic. + */ +public class StateFunction implements Function { + + @Override + public String process(String input, Context context) { + Map userConfigMap = context.getUserConfigMap(); + if(userConfigMap.containsKey("statekey")){ + String statekey = userConfigMap.get("statekey").toString(); + context.putState(statekey,ByteBuffer.wrap(input.getBytes())); + } + if(userConfigMap.containsKey("incrkey")) { + String incrkey = userConfigMap.get("incrkey").toString(); + context.incrCounter(incrkey,1); + } + return String.format("%s!", input); + } + +} diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index e5baa7ae9671a..89f04dcc3220b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -128,9 +128,13 @@ public static List getArgsBeforeCmd(InstanceConfig instanceConfig, Strin */ public static List getGoInstanceCmd(InstanceConfig instanceConfig, + AuthenticationConfig authConfig, String originalCodeFileName, String pulsarServiceUrl, boolean k8sRuntime) throws IOException { + + + final List args = new LinkedList<>(); GoInstanceConfig goInstanceConfig = new GoInstanceConfig(); @@ -243,6 +247,28 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig, goInstanceConfig.setMetricsPort(instanceConfig.getMetricsPort()); } + if (authConfig != null) { + if (isNotBlank(authConfig.getClientAuthenticationPlugin()) + && isNotBlank(authConfig.getClientAuthenticationParameters())) { +// args.add("--clientAuthPlugin"); +// args.add(authConfig.getClientAuthenticationPlugin()); +// args.add("--clientAuthParams"); +// args.add(authConfig.getClientAuthenticationParameters()); + goInstanceConfig.setClientAuthPlugin(authConfig.getClientAuthenticationPlugin()); + goInstanceConfig.setClientAuthParams(authConfig.getClientAuthenticationParameters()); + } +// args.add("--use_tls"); +// args.add(Boolean.toString(authConfig.isUseTls())); +// args.add("--tls_allow_insecure"); +// args.add(Boolean.toString(authConfig.isTlsAllowInsecureConnection())); +// args.add("--hostname_verification_enabled"); +// args.add(Boolean.toString(authConfig.isTlsHostnameVerificationEnable())); +// if (isNotBlank(authConfig.getTlsTrustCertsFilePath())) { +// args.add("--tls_trust_cert_path"); +// args.add(authConfig.getTlsTrustCertsFilePath()); +// } + } + goInstanceConfig.setKillAfterIdleMs(0); goInstanceConfig.setPort(instanceConfig.getPort()); @@ -285,7 +311,7 @@ public static List getCmd(InstanceConfig instanceConfig, final List args = new LinkedList<>(); if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.GO) { - return getGoInstanceCmd(instanceConfig, originalCodeFileName, pulsarServiceUrl, k8sRuntime); + return getGoInstanceCmd(instanceConfig,authConfig, originalCodeFileName, pulsarServiceUrl, k8sRuntime); } if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.JAVA) { diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java index f7cdf707d3219..3bf3bb2d09399 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java @@ -116,7 +116,7 @@ public void getGoInstanceCmd(boolean k8sRuntime) throws IOException { instanceConfig.setFunctionDetails(functionDetails); - List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, "config", "pulsar://localhost:6650", k8sRuntime); + List commands = RuntimeUtils.getGoInstanceCmd(instanceConfig, null,"config", "pulsar://localhost:6650", k8sRuntime); if (k8sRuntime) { goInstanceConfig = new ObjectMapper().readValue(commands.get(2).replaceAll("^\'|\'$", ""), HashMap.class); } else { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 96c97b31e51b0..eecba36f94f5e 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -268,6 +268,7 @@ public final CompletableFuture> getChildren(String path) { if (!isValidPath(path)) { return FutureUtil.failedFuture(new MetadataStoreException.InvalidPathException(path)); } +// log.info("lan.dev.childrenCache:{}",childrenCache); return childrenCache.get(path); } @@ -394,6 +395,7 @@ protected abstract CompletableFuture storePut(String path, byte[] data, Op @Override public final CompletableFuture put(String path, byte[] data, Optional optExpectedVersion, EnumSet options) { + //lan.dev /managed-ledgers/ public/default/test /persistent/test3 long start = System.currentTimeMillis(); if (!isValidPath(path)) { metadataStoreStats.recordPutOpsFailed(System.currentTimeMillis() - start); diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 452c98fbb6039..e7c9fd231d9de 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -77,6 +77,16 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + 17 + 17 + UTF-8 + + org.apache.maven.plugins maven-checkstyle-plugin