diff --git a/Pipfile b/Pipfile index 1f5f964..bde555b 100644 --- a/Pipfile +++ b/Pipfile @@ -7,10 +7,11 @@ name = "pypi" attrs = "*" beautifulsoup4 = "*" click = "*" +jsonlines = "*" lxml = "*" +python-dateutil = "*" sentry-sdk = "*" smart-open = {version = "*", extras = ["s3"]} -python-dateutil = "*" types-python-dateutil = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 5ffef10..218e248 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d8d1a9284afa82486c23db55536a78974489321be96ca82fe6ffc8fcc3a4ee00" + "sha256": "0fca61ae276dec68a93d70d8e67a533dc08d8ab5cc6a4c5fe036ce4d90e5ec79" }, "pipfile-spec": 6, "requires": { @@ -36,18 +36,18 @@ }, "boto3": { "hashes": [ - "sha256:ca9b04fc2c75990c2be84c43b9d6edecce828960fc27e07ab29036587a1ca635", - "sha256:d1135647309b89376a014d21407aabfa322998206175f2297def812bf4d824a9" + "sha256:8b3c4d4e720c0ad706590c284b8f30c76de3472c1ce1bac610425f99bf6ab53b", + "sha256:c9b400529932ed4652304756528ab235c6730aa5d00cb4d9e4848ce460c82c16" ], - "version": "==1.29.4" + "version": "==1.34.0" }, "botocore": { "hashes": [ - "sha256:3ee73c0d93bdb944d0c46772f08f09cdcf25ef58bd86962e6f4a24e531198bfa", - "sha256:6bfa75e28c9ad0321cefefa51b00ff233b16b2416f8b95229796263edba45a39" + "sha256:6ec19f6c9f61c3df22fb3e083940ac7946a3d96128db1f370f10aea702bb157f", + "sha256:711b406de910585395466ca649bceeea87a04300ddf74d9a2e20727c7f27f2f1" ], - "markers": "python_version >= '3.7'", - "version": "==1.32.4" + "markers": "python_version >= '3.8'", + "version": "==1.34.0" }, "certifi": { "hashes": [ @@ -74,6 +74,15 @@ "markers": "python_version >= '3.7'", "version": "==1.0.1" }, + "jsonlines": { + "hashes": [ + "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74", + "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==4.0.0" + }, "lxml": { "hashes": [ "sha256:05186a0f1346ae12553d66df1cfce6f251589fea3ad3da4f3ef4e34b2d58c6a3", @@ -184,19 +193,19 @@ }, "s3transfer": { "hashes": [ - "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a", - "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e" + "sha256:01d4d2c35a016db8cb14f9a4d5e84c1f8c96e7ffc211422555eed45c11fa7eb1", + "sha256:9e1b186ec8bb5907a1e82b51237091889a9973a2bb799a924bcd9f301ff79d3d" ], - "markers": "python_version >= '3.7'", - "version": "==0.7.0" + "markers": "python_version >= '3.8'", + "version": "==0.9.0" }, "sentry-sdk": { "hashes": [ - "sha256:25d574f94fdf72199e331c2401fdac60d01b5be8f32822174c51c3ff0fc2f8cb", - "sha256:f32dd16547f2f45e1c71a96fd4a48925e629541f7ddfe3d5d25ef7d5e94eb3c8" + "sha256:320a55cdf9da9097a0bead239c35b7e61f53660ef9878861824fd6d9b2eaf3b5", + "sha256:81b5b9ffdd1a374e9eb0c053b5d2012155db9cbe76393a8585677b753bd5fdc1" ], "index": "pypi", - "version": "==1.36.0" + "version": "==1.39.1" }, "six": { "hashes": [ @@ -243,13 +252,94 @@ } }, "develop": { - "appnope": { - "hashes": [ - "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", - "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" + "aiohttp": { + "hashes": [ + "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f", + "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c", + "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af", + "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4", + "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a", + "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489", + "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213", + "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01", + "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5", + "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361", + "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26", + "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0", + "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4", + "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8", + "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1", + "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7", + "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6", + "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a", + "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd", + "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4", + "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499", + "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183", + "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544", + "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821", + "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501", + "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f", + "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe", + "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f", + "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672", + "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5", + "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2", + "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57", + "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87", + "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0", + "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f", + "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7", + "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed", + "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70", + "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0", + "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f", + "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d", + "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f", + "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d", + "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431", + "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff", + "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf", + "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83", + "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690", + "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587", + "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e", + "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb", + "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3", + "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66", + "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014", + "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35", + "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f", + "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0", + "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449", + "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23", + "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5", + "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd", + "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4", + "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b", + "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558", + "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd", + "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766", + "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a", + "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636", + "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d", + "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590", + "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e", + "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d", + "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c", + "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28", + "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065", + "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca" + ], + "version": "==3.9.1" + }, + "aiosignal": { + "hashes": [ + "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", + "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17" ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.3" + "markers": "python_version >= '3.7'", + "version": "==1.3.1" }, "asttokens": { "hashes": [ @@ -258,39 +348,60 @@ ], "version": "==2.4.1" }, - "bandit": { + "async-timeout": { + "hashes": [ + "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", + "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028" + ], + "markers": "python_version < '3.11'", + "version": "==4.0.3" + }, + "attrs": { "hashes": [ - "sha256:75665181dc1e0096369112541a056c59d1c5f66f9bb74a8d686c3c362b83f549", - "sha256:bdfc739baa03b880c2d15d0431b31c658ffc348e907fe197e54e0389dd59e11e" + "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", + "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.7.5" + "version": "==23.1.0" + }, + "bandit": { + "hashes": [ + "sha256:36da17c67fc87579a5d20c323c8d0b1643a890a2b93f00b3d1229966624694ff", + "sha256:72ce7bc9741374d96fb2f1c9a8960829885f1243ffde743de70a19cee353e8f3" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.7.6" }, "black": { "hashes": [ - "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4", - "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b", - "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f", - "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07", - "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187", - "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6", - "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05", - "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06", - "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e", - "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5", - "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244", - "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f", - "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221", - "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055", - "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479", - "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394", - "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911", - "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142" + "sha256:12d5f10cce8dc27202e9a252acd1c9a426c83f95496c959406c96b785a92bb7d", + "sha256:193946e634e80bfb3aec41830f5d7431f8dd5b20d11d89be14b84a97c6b8bc75", + "sha256:330a327b422aca0634ecd115985c1c7fd7bdb5b5a2ef8aa9888a82e2ebe9437a", + "sha256:39dda060b9b395a6b7bf9c5db28ac87b3c3f48d4fdff470fa8a94ab8271da47e", + "sha256:593596f699ca2dcbbbdfa59fcda7d8ad6604370c10228223cd6cf6ce1ce7ed7e", + "sha256:67f19562d367468ab59bd6c36a72b2c84bc2f16b59788690e02bbcb140a77175", + "sha256:6a82a711d13e61840fb11a6dfecc7287f2424f1ca34765e70c909a35ffa7fb95", + "sha256:7231670266ca5191a76cb838185d9be59cfa4f5dd401b7c1c70b993c58f6b1b5", + "sha256:72db37a2266b16d256b3ea88b9affcdd5c41a74db551ec3dd4609a59c17d25bf", + "sha256:81a832b6e00eef2c13b3239d514ea3b7d5cc3eaa03d0474eedcbbda59441ba5d", + "sha256:97af22278043a6a1272daca10a6f4d36c04dfa77e61cbaaf4482e08f3640e9f0", + "sha256:996650a89fe5892714ea4ea87bc45e41a59a1e01675c42c433a35b490e5aa3f0", + "sha256:a7c07db8200b5315dc07e331dda4d889a56f6bf4db6a9c2a526fa3166a81614f", + "sha256:ace64c1a349c162d6da3cef91e3b0e78c4fc596ffde9413efa0525456148873d", + "sha256:ba09cae1657c4f8a8c9ff6cfd4a6baaf915bb4ef7d03acffe6a2f6585fa1bd01", + "sha256:bbd75d9f28a7283b7426160ca21c5bd640ca7cd8ef6630b4754b6df9e2da8462", + "sha256:bcf91b01ddd91a2fed9a8006d7baa94ccefe7e518556470cf40213bd3d44bbbc", + "sha256:bdbff34c487239a63d86db0c9385b27cdd68b1bfa4e706aa74bb94a435403672", + "sha256:c71048345bdbced456cddf1622832276d98a710196b842407840ae8055ade6ee", + "sha256:e73c5e3d37e5a3513d16b33305713237a234396ae56769b839d7c40759b8a41c", + "sha256:ead25c273adfad1095a8ad32afdb8304933efba56e3c1d31b0fee4143a1e424a", + "sha256:fdf6f23c83078a6c8da2442f4d4eeb19c28ac2a6416da7671b72f0295c4a697b" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==23.11.0" + "version": "==23.12.0" }, "certifi": { "hashes": [ @@ -510,6 +621,73 @@ "markers": "python_full_version >= '3.8.1'", "version": "==6.1.0" }, + "frozenlist": { + "hashes": [ + "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6", + "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01", + "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251", + "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9", + "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b", + "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87", + "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf", + "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f", + "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0", + "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2", + "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b", + "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc", + "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c", + "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467", + "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9", + "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1", + "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a", + "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79", + "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167", + "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300", + "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf", + "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea", + "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2", + "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab", + "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3", + "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb", + "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087", + "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc", + "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8", + "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62", + "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f", + "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326", + "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c", + "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431", + "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963", + "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7", + "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef", + "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3", + "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956", + "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781", + "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472", + "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc", + "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839", + "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672", + "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3", + "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503", + "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d", + "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8", + "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b", + "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc", + "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f", + "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559", + "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b", + "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95", + "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb", + "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963", + "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919", + "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f", + "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3", + "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1", + "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e" + ], + "markers": "python_version >= '3.8'", + "version": "==1.4.0" + }, "gitdb": { "hashes": [ "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4", @@ -528,11 +706,11 @@ }, "idna": { "hashes": [ - "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", - "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", + "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" ], "markers": "python_version >= '3.5'", - "version": "==3.4" + "version": "==3.6" }, "iniconfig": { "hashes": [ @@ -544,21 +722,21 @@ }, "ipython": { "hashes": [ - "sha256:126bb57e1895594bb0d91ea3090bbd39384f6fe87c3d57fd558d0670f50339bb", - "sha256:1e4d1d666a023e3c93585ba0d8e962867f7a111af322efff6b9c58062b3e5444" + "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27", + "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==8.17.2" + "version": "==8.18.1" }, "isort": { "hashes": [ - "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504", - "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6" + "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", + "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" ], "index": "pypi", "markers": "python_full_version >= '3.8.0'", - "version": "==5.12.0" + "version": "==5.13.2" }, "jedi": { "hashes": [ @@ -600,39 +778,119 @@ "markers": "python_version >= '3.7'", "version": "==0.1.2" }, + "multidict": { + "hashes": [ + "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9", + "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8", + "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03", + "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710", + "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161", + "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664", + "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569", + "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067", + "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313", + "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706", + "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2", + "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636", + "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49", + "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93", + "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603", + "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0", + "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60", + "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4", + "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e", + "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1", + "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60", + "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951", + "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc", + "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe", + "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95", + "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d", + "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8", + "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed", + "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2", + "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775", + "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87", + "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c", + "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2", + "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98", + "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3", + "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe", + "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78", + "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660", + "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176", + "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e", + "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988", + "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c", + "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c", + "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0", + "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449", + "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f", + "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde", + "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5", + "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d", + "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac", + "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a", + "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9", + "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca", + "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11", + "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35", + "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063", + "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b", + "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982", + "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258", + "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1", + "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52", + "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480", + "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7", + "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461", + "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d", + "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc", + "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779", + "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a", + "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547", + "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0", + "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171", + "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf", + "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d", + "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba" + ], + "markers": "python_version >= '3.7'", + "version": "==6.0.4" + }, "mypy": { "hashes": [ - "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46", - "sha256:185cff9b9a7fec1f9f7d8352dff8a4c713b2e3eea9c6c4b5ff7f0edf46b91e41", - "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc", - "sha256:1fe46e96ae319df21359c8db77e1aecac8e5949da4773c0274c0ef3d8d1268a9", - "sha256:2b53655a295c1ed1af9e96b462a736bf083adba7b314ae775563e3fb4e6795f5", - "sha256:551d4a0cdcbd1d2cccdcc7cb516bb4ae888794929f5b040bb51aae1846062901", - "sha256:55d28d7963bef00c330cb6461db80b0b72afe2f3c4e2963c99517cf06454e665", - "sha256:5da84d7bf257fd8f66b4f759a904fd2c5a765f70d8b52dde62b521972a0a2357", - "sha256:6cb8d5f6d0fcd9e708bb190b224089e45902cacef6f6915481806b0c77f7786d", - "sha256:7a7b1e399c47b18feb6f8ad4a3eef3813e28c1e871ea7d4ea5d444b2ac03c418", - "sha256:870bd1ffc8a5862e593185a4c169804f2744112b4a7c55b93eb50f48e7a77010", - "sha256:87c076c174e2c7ef8ab416c4e252d94c08cd4980a10967754f91571070bf5fbe", - "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96", - "sha256:a3637c03f4025f6405737570d6cbfa4f1400eb3c649317634d273687a09ffc2f", - "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d", - "sha256:b633f188fc5ae1b6edca39dae566974d7ef4e9aaaae00bc36efe1f855e5173ac", - "sha256:bf7a2f0a6907f231d5e41adba1a82d7d88cf1f61a70335889412dec99feeb0f8", - "sha256:c1b06b4b109e342f7dccc9efda965fc3970a604db70f8560ddfdee7ef19afb05", - "sha256:cddee95dea7990e2215576fae95f6b78a8c12f4c089d7e4367564704e99118d3", - "sha256:d01921dbd691c4061a3e2ecdbfbfad029410c5c2b1ee88946bf45c62c6c91210", - "sha256:d0fa29919d2e720c8dbaf07d5578f93d7b313c3e9954c8ec05b6d83da592e5d9", - "sha256:d6ed9a3997b90c6f891138e3f83fb8f475c74db4ccaa942a1c7bf99e83a989a1", - "sha256:d93e76c2256aa50d9c82a88e2f569232e9862c9982095f6d54e13509f01222fc", - "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401", - "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee", - "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1", - "sha256:fc9fe455ad58a20ec68599139ed1113b21f977b536a91b42bef3ffed5cce7391" + "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340", + "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49", + "sha256:204e0d6de5fd2317394a4eff62065614c4892d5a4d1a7ee55b765d7a3d9e3f82", + "sha256:2643d145af5292ee956aa0a83c2ce1038a3bdb26e033dadeb2f7066fb0c9abce", + "sha256:2c6e4464ed5f01dc44dc9821caf67b60a4e5c3b04278286a85c067010653a0eb", + "sha256:2f7f6985d05a4e3ce8255396df363046c28bea790e40617654e91ed580ca7c51", + "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5", + "sha256:40716d1f821b89838589e5b3106ebbc23636ffdef5abc31f7cd0266db936067e", + "sha256:4b901927f16224d0d143b925ce9a4e6b3a758010673eeded9b748f250cf4e8f7", + "sha256:4fc3d14ee80cd22367caaaf6e014494415bf440980a3045bf5045b525680ac33", + "sha256:5cf3f0c5ac72139797953bd50bc6c95ac13075e62dbfcc923571180bebb662e9", + "sha256:6dbdec441c60699288adf051f51a5d512b0d818526d1dcfff5a41f8cd8b4aaf1", + "sha256:72cf32ce7dd3562373f78bd751f73c96cfb441de147cc2448a92c1a308bd0ca6", + "sha256:75aa828610b67462ffe3057d4d8a4112105ed211596b750b53cbfe182f44777a", + "sha256:75c4d2a6effd015786c87774e04331b6da863fc3fc4e8adfc3b40aa55ab516fe", + "sha256:78e25b2fd6cbb55ddfb8058417df193f0129cad5f4ee75d1502248e588d9e0d7", + "sha256:84860e06ba363d9c0eeabd45ac0fde4b903ad7aa4f93cd8b648385a888e23200", + "sha256:8c5091ebd294f7628eb25ea554852a52058ac81472c921150e3a61cdd68f75a7", + "sha256:944bdc21ebd620eafefc090cdf83158393ec2b1391578359776c00de00e8907a", + "sha256:9c7ac372232c928fff0645d85f273a726970c014749b924ce5710d7d89763a28", + "sha256:d9b338c19fa2412f76e17525c1b4f2c687a55b156320acb588df79f2e6fa9fea", + "sha256:ee5d62d28b854eb61889cde4e1dbc10fbaa5560cb39780c3995f6737f7e82120", + "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d", + "sha256:f6efc9bd72258f89a3816e3a98c09d36f079c223aa345c659622f056b760ab42", + "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea", + "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2", + "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.7.0" + "version": "==1.7.1" }, "mypy-extensions": { "hashes": [ @@ -660,11 +918,11 @@ }, "pathspec": { "hashes": [ - "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20", - "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3" + "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", + "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712" ], - "markers": "python_version >= '3.7'", - "version": "==0.11.2" + "markers": "python_version >= '3.8'", + "version": "==0.12.1" }, "pbr": { "hashes": [ @@ -676,19 +934,19 @@ }, "pexpect": { "hashes": [ - "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", - "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" + "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", + "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f" ], "markers": "sys_platform != 'win32'", - "version": "==4.8.0" + "version": "==4.9.0" }, "platformdirs": { "hashes": [ - "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b", - "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731" + "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380", + "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420" ], - "markers": "python_version >= '3.7'", - "version": "==4.0.0" + "markers": "python_version >= '3.8'", + "version": "==4.1.0" }, "pluggy": { "hashes": [ @@ -700,11 +958,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0", - "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2" + "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d", + "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.0.41" + "version": "==3.0.43" }, "ptyprocess": { "hashes": [ @@ -738,11 +996,11 @@ }, "pygments": { "hashes": [ - "sha256:1b37f1b1e1bff2af52ecaf28cc601e2ef7077000b227a0675da25aef85784bc4", - "sha256:e45a0e74bf9c530f564ca81b8952343be986a29f6afe7f5ad95c5f06b7bdf5e8" + "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c", + "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367" ], "markers": "python_version >= '3.7'", - "version": "==2.17.1" + "version": "==2.17.2" }, "pytest": { "hashes": [ @@ -866,19 +1124,19 @@ }, "traitlets": { "hashes": [ - "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5", - "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619" + "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33", + "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772" ], "markers": "python_version >= '3.8'", - "version": "==5.13.0" + "version": "==5.14.0" }, "typing-extensions": { "hashes": [ - "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", - "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef" + "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", + "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" ], "markers": "python_version >= '3.8'", - "version": "==4.8.0" + "version": "==4.9.0" }, "urllib3": { "hashes": [ @@ -890,10 +1148,106 @@ }, "wcwidth": { "hashes": [ - "sha256:25eb3ecbec328cdb945f56f2a7cfe784bdf7a73a8197398c7a7c65e7fe93e9ae", - "sha256:c4b153acf29f1f0d7fb1b00d097cce82b73de7a2016321c8d7ca71bd76dd848b" + "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02", + "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c" + ], + "version": "==0.2.12" + }, + "yarl": { + "hashes": [ + "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51", + "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce", + "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559", + "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0", + "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81", + "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc", + "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4", + "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c", + "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130", + "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136", + "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e", + "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec", + "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7", + "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1", + "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455", + "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099", + "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129", + "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10", + "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142", + "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98", + "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa", + "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7", + "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525", + "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c", + "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9", + "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c", + "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8", + "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b", + "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf", + "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23", + "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd", + "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27", + "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f", + "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece", + "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434", + "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec", + "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff", + "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78", + "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d", + "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863", + "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53", + "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31", + "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15", + "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5", + "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b", + "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57", + "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3", + "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1", + "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f", + "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad", + "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c", + "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7", + "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2", + "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b", + "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2", + "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b", + "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9", + "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be", + "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e", + "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984", + "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4", + "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074", + "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2", + "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392", + "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91", + "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541", + "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf", + "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572", + "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66", + "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575", + "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14", + "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5", + "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1", + "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e", + "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551", + "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17", + "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead", + "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0", + "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe", + "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234", + "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0", + "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7", + "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34", + "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42", + "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385", + "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78", + "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be", + "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958", + "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749", + "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec" ], - "version": "==0.2.11" + "markers": "python_version >= '3.7'", + "version": "==1.9.4" } } } diff --git a/tests/conftest.py b/tests/conftest.py index cd12279..8616c13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,8 +5,8 @@ import transmogrifier.models as timdex from transmogrifier.config import SOURCES, load_external_config -from transmogrifier.sources.datacite import Datacite -from transmogrifier.sources.transformer import XmlTransformer +from transmogrifier.sources.transformer import JsonTransformer, XmlTransformer +from transmogrifier.sources.xml.datacite import Datacite @pytest.fixture(autouse=True) @@ -29,7 +29,7 @@ def bad_config(): SOURCES["bad-class-name"] = { "name": "Some Repository", "base-url": "https://example.com/", - "transform-class": "transmogrifier.sources.datacite.WrongClass", + "transform-class": "transmogrifier.sources.xml.datacite.WrongClass", } SOURCES["bad-module-path"] = { "name": "Some Repository", @@ -46,6 +46,15 @@ def runner(): return CliRunner() +@pytest.fixture +def aardvark_record_all_fields(): + return next( + JsonTransformer.parse_source_file( + "tests/fixtures/aardvark/aardvark_record_all_fields.jsonl" + ) + ) + + @pytest.fixture() def datacite_records(): return XmlTransformer.parse_source_file( @@ -61,6 +70,11 @@ def datacite_record_all_fields(): return Datacite("cool-repo", source_records) +@pytest.fixture() +def aardvark_records(): + return JsonTransformer.parse_source_file("tests/fixtures/aardvark_records.jsonl") + + @pytest.fixture() def loc_country_crosswalk(): return load_external_config("config/loc-countries.xml", "xml") diff --git a/tests/fixtures/aardvark/aardvark_record_all_fields.jsonl b/tests/fixtures/aardvark/aardvark_record_all_fields.jsonl new file mode 100644 index 0000000..af39020 --- /dev/null +++ b/tests/fixtures/aardvark/aardvark_record_all_fields.jsonl @@ -0,0 +1 @@ +{"id": "123", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "dct_title_s": "Test title 1"} \ No newline at end of file diff --git a/tests/fixtures/aardvark_records.jsonl b/tests/fixtures/aardvark_records.jsonl new file mode 100644 index 0000000..2502b07 --- /dev/null +++ b/tests/fixtures/aardvark_records.jsonl @@ -0,0 +1,2 @@ +{"id": "123", "dct_title_s": "Test title 1"} +{"id": "456", "dct_title_s": "Test title 2"} \ No newline at end of file diff --git a/tests/sources/json/test_aardvark.py b/tests/sources/json/test_aardvark.py new file mode 100644 index 0000000..a87637e --- /dev/null +++ b/tests/sources/json/test_aardvark.py @@ -0,0 +1,43 @@ +import transmogrifier.models as timdex +from transmogrifier.sources.json.aardvark import MITAardvark + + +def test_aardvark_get_required_fields_returns_expected_values(aardvark_records): + transformer = MITAardvark("cool-repo", aardvark_records) + assert transformer.get_required_fields(next(aardvark_records)) == { + "source": "A Cool Repository", + "source_link": "https://example.com/123", + "timdex_record_id": "cool-repo:123", + "title": "Test title 1", + } + + +def test_jsontransformer_transform_returns_timdex_record(aardvark_records): + transformer = MITAardvark("cool-repo", aardvark_records) + assert next(transformer) == timdex.TimdexRecord( + source="A Cool Repository", + source_link="https://example.com/123", + timdex_record_id="cool-repo:123", + title="Test title 1", + citation="Test title 1. Geospatial data. https://example.com/123", + content_type=["Geospatial data"], + ) + + +def test_aardvark_get_main_titles_success(aardvark_record_all_fields): + assert MITAardvark.get_main_titles(aardvark_record_all_fields) == ["Test title 1"] + + +def test_aardvark_get_source_record_id_success(aardvark_record_all_fields): + assert MITAardvark.get_source_record_id(aardvark_record_all_fields) == "123" + + +def test_aardvark_get_subjects_success(aardvark_record_all_fields): + assert MITAardvark.get_subjects(aardvark_record_all_fields) == [ + timdex.Subject(value=["Country"], kind="DCAT Keyword"), + timdex.Subject(value=["Political boundaries"], kind="DCAT Theme"), + timdex.Subject(value=["Geography"], kind="Dublin Core Subject"), + timdex.Subject(value=["Earth"], kind="Dublin Core Subject"), + timdex.Subject(value=["Dataset"], kind="Subject scheme not provided"), + timdex.Subject(value=["Vector data"], kind="Subject scheme not provided"), + ] diff --git a/tests/test_transformer.py b/tests/sources/test_transformer.py similarity index 94% rename from tests/test_transformer.py rename to tests/sources/test_transformer.py index 3a0ddb9..33ecb84 100644 --- a/tests/test_transformer.py +++ b/tests/sources/test_transformer.py @@ -1,10 +1,11 @@ +from pathlib import Path from unittest.mock import patch import pytest from transmogrifier.models import TimdexRecord -from transmogrifier.sources.datacite import Datacite from transmogrifier.sources.transformer import Transformer, XmlTransformer +from transmogrifier.sources.xml.datacite import Datacite def test_transformer_get_transformer_returns_correct_class_name(): @@ -62,11 +63,11 @@ def test_xmltransformer_transform_and_write_output_files_writes_output_files( ): output_file = str(tmp_path / "output_file.json") transformer = XmlTransformer("cool-repo", oai_pmh_records) + assert not Path(tmp_path / "output_file.json").exists() + assert not Path(tmp_path / "output_file.txt").exists() transformer.transform_and_write_output_files(output_file) - output_files = list(tmp_path.iterdir()) - assert len(output_files) == 2 - assert output_files[0].name == "output_file.json" - assert output_files[1].name == "output_file.txt" + assert Path(tmp_path / "output_file.json").exists() + assert Path(tmp_path / "output_file.txt").exists() def test_xmltransformer_transform_and_write_output_files_no_txt_file_if_not_needed( diff --git a/tests/test_datacite.py b/tests/sources/xml/test_datacite.py similarity index 99% rename from tests/test_datacite.py rename to tests/sources/xml/test_datacite.py index 661ce5e..02d47f6 100644 --- a/tests/test_datacite.py +++ b/tests/sources/xml/test_datacite.py @@ -13,7 +13,7 @@ Subject, TimdexRecord, ) -from transmogrifier.sources.datacite import Datacite +from transmogrifier.sources.xml.datacite import Datacite def test_datacite_transform_with_all_fields_transforms_correctly( diff --git a/tests/test_dspace_dim.py b/tests/sources/xml/test_dspace_dim.py similarity index 99% rename from tests/test_dspace_dim.py rename to tests/sources/xml/test_dspace_dim.py index b86c9df..45f7177 100644 --- a/tests/test_dspace_dim.py +++ b/tests/sources/xml/test_dspace_dim.py @@ -1,5 +1,5 @@ import transmogrifier.models as timdex -from transmogrifier.sources.dspace_dim import DspaceDim +from transmogrifier.sources.xml.dspace_dim import DspaceDim def test_dspace_dim_transform_with_all_fields_transforms_correctly(): diff --git a/tests/test_dspace_mets.py b/tests/sources/xml/test_dspace_mets.py similarity index 99% rename from tests/test_dspace_mets.py rename to tests/sources/xml/test_dspace_mets.py index fb7b8bc..378e01f 100644 --- a/tests/test_dspace_mets.py +++ b/tests/sources/xml/test_dspace_mets.py @@ -1,5 +1,5 @@ import transmogrifier.models as timdex -from transmogrifier.sources.dspace_mets import DspaceMets +from transmogrifier.sources.xml.dspace_mets import DspaceMets def test_dspace_mets_transform_with_missing_optional_fields_transforms_correctly(): diff --git a/tests/test_ead.py b/tests/sources/xml/test_ead.py similarity index 99% rename from tests/test_ead.py rename to tests/sources/xml/test_ead.py index c7de237..d0c91e1 100644 --- a/tests/test_ead.py +++ b/tests/sources/xml/test_ead.py @@ -1,7 +1,7 @@ import logging import transmogrifier.models as timdex -from transmogrifier.sources.ead import Ead +from transmogrifier.sources.xml.ead import Ead def test_ead_record_all_fields_transform_correctly(): @@ -224,7 +224,7 @@ def test_ead_record_with_missing_archdesc_logs_error(caplog): assert len(list(output_records)) == 0 assert output_records.processed_record_count == 1 assert ( - "transmogrifier.sources.ead", + "transmogrifier.sources.xml.ead", logging.ERROR, "Record ID repositories/2/resources/4 is missing archdesc element", ) in caplog.record_tuples @@ -238,7 +238,7 @@ def test_ead_record_with_missing_archdesc_did_logs_error(caplog): assert len(list(output_records)) == 0 assert output_records.processed_record_count == 1 assert ( - "transmogrifier.sources.ead", + "transmogrifier.sources.xml.ead", logging.ERROR, "Record ID repositories/2/resources/3 is missing archdesc > did element", ) in caplog.record_tuples diff --git a/tests/test_marc.py b/tests/sources/xml/test_marc.py similarity index 99% rename from tests/test_marc.py rename to tests/sources/xml/test_marc.py index 2256dbc..c539c88 100644 --- a/tests/test_marc.py +++ b/tests/sources/xml/test_marc.py @@ -3,7 +3,7 @@ from bs4 import BeautifulSoup import transmogrifier.models as timdex -from transmogrifier.sources.marc import Marc +from transmogrifier.sources.xml.marc import Marc def test_marc_record_all_fields_transform_correctly(): @@ -752,7 +752,7 @@ def test_marc_record_missing_leader_logs_error(caplog): assert len(list(output_records)) == 0 assert output_records.processed_record_count == 1 assert ( - "transmogrifier.sources.marc", + "transmogrifier.sources.xml.marc", logging.ERROR, "Record ID 990027185640106761 is missing MARC leader", ) in caplog.record_tuples @@ -766,7 +766,7 @@ def test_marc_record_missing_008_logs_error(caplog): assert len(list(output_records)) == 0 assert output_records.processed_record_count == 1 assert ( - "transmogrifier.sources.marc", + "transmogrifier.sources.xml.marc", logging.ERROR, "Record ID 990027185640106761 is missing MARC 008 field", ) in caplog.record_tuples diff --git a/tests/test_oai_dc.py b/tests/sources/xml/test_oai_dc.py similarity index 98% rename from tests/test_oai_dc.py rename to tests/sources/xml/test_oai_dc.py index d3103f0..3fc6eaa 100644 --- a/tests/test_oai_dc.py +++ b/tests/sources/xml/test_oai_dc.py @@ -1,5 +1,5 @@ import transmogrifier.models as timdex -from transmogrifier.sources.oaidc import OaiDc +from transmogrifier.sources.xml.oaidc import OaiDc FIXTURES_PREFIX = "tests/fixtures/oai_dc" diff --git a/tests/test_springshare.py b/tests/sources/xml/test_springshare.py similarity index 99% rename from tests/test_springshare.py rename to tests/sources/xml/test_springshare.py index 67a81e4..62f6d04 100644 --- a/tests/test_springshare.py +++ b/tests/sources/xml/test_springshare.py @@ -1,5 +1,5 @@ import transmogrifier.models as timdex -from transmogrifier.sources.springshare import SpringshareOaiDc +from transmogrifier.sources.xml.springshare import SpringshareOaiDc SPRINGSHARE_FIXTURES_PREFIX = "tests/fixtures/oai_dc/springshare" diff --git a/tests/test_whoas.py b/tests/sources/xml/test_whoas.py similarity index 95% rename from tests/test_whoas.py rename to tests/sources/xml/test_whoas.py index 29d0664..a227a42 100644 --- a/tests/test_whoas.py +++ b/tests/sources/xml/test_whoas.py @@ -1,4 +1,4 @@ -from transmogrifier.sources.whoas import Whoas +from transmogrifier.sources.xml.whoas import Whoas def test_valid_content_types_with_all_invalid(): diff --git a/tests/test_zenodo.py b/tests/sources/xml/test_zenodo.py similarity index 96% rename from tests/test_zenodo.py rename to tests/sources/xml/test_zenodo.py index 5589e51..8792e92 100644 --- a/tests/test_zenodo.py +++ b/tests/sources/xml/test_zenodo.py @@ -1,4 +1,4 @@ -from transmogrifier.sources.zenodo import Zenodo +from transmogrifier.sources.xml.zenodo import Zenodo def test_zenodo_create_source_record_id_generates_correct_id(): diff --git a/transmogrifier/config.py b/transmogrifier/config.py index ea96c16..05f8c10 100644 --- a/transmogrifier/config.py +++ b/transmogrifier/config.py @@ -81,42 +81,42 @@ "https://mit.primo.exlibrisgroup.com/discovery/fulldisplay?" "vid=01MIT_INST:MIT&docid=alma" ), - "transform-class": "transmogrifier.sources.marc.Marc", + "transform-class": "transmogrifier.sources.xml.marc.Marc", }, "aspace": { "name": "MIT ArchivesSpace", "base-url": "https://archivesspace.mit.edu/", - "transform-class": "transmogrifier.sources.ead.Ead", + "transform-class": "transmogrifier.sources.xml.ead.Ead", }, "dspace": { "name": "DSpace@MIT", "base-url": "https://dspace.mit.edu/handle/", - "transform-class": "transmogrifier.sources.dspace_mets.DspaceMets", + "transform-class": "transmogrifier.sources.xml.dspace_mets.DspaceMets", }, "jpal": { "name": "Abdul Latif Jameel Poverty Action Lab Dataverse", "base-url": "https://dataverse.harvard.edu/dataset.xhtml?persistentId=", - "transform-class": "transmogrifier.sources.datacite.Datacite", + "transform-class": "transmogrifier.sources.xml.datacite.Datacite", }, "libguides": { "name": "LibGuides", "base-url": "https://libguides.mit.edu/", - "transform-class": "transmogrifier.sources.springshare.SpringshareOaiDc", + "transform-class": "transmogrifier.sources.xml.springshare.SpringshareOaiDc", }, "researchdatabases": { "name": "Research Databases", "base-url": "https://libguides.mit.edu/", - "transform-class": "transmogrifier.sources.springshare.SpringshareOaiDc", + "transform-class": "transmogrifier.sources.xml.springshare.SpringshareOaiDc", }, "whoas": { "name": "Woods Hole Open Access Server", "base-url": "https://darchive.mblwhoilibrary.org/handle/", - "transform-class": "transmogrifier.sources.whoas.Whoas", + "transform-class": "transmogrifier.sources.xml.whoas.Whoas", }, "zenodo": { "name": "Zenodo", "base-url": "https://zenodo.org/record/", - "transform-class": "transmogrifier.sources.zenodo.Zenodo", + "transform-class": "transmogrifier.sources.xml.zenodo.Zenodo", }, } diff --git a/transmogrifier/sources/json/__init__.py b/transmogrifier/sources/json/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transmogrifier/sources/json/aardvark.py b/transmogrifier/sources/json/aardvark.py new file mode 100644 index 0000000..25429cf --- /dev/null +++ b/transmogrifier/sources/json/aardvark.py @@ -0,0 +1,132 @@ +import logging + +import transmogrifier.models as timdex +from transmogrifier.sources.transformer import JsonTransformer + +logger = logging.getLogger(__name__) + + +class MITAardvark(JsonTransformer): + """MITAardvark transformer. + + MIT Aardvark records have more required fields than standard Aardvark records + as detailed here in the geo-harvester's MITAardvark class: + + https://github.com/MITLibraries/geo-harvester/blob/main/harvester/records/record.py + """ + + @classmethod + def get_main_titles(cls, source_record: dict) -> list[str]: + """ + Retrieve main title(s) from a MITAardvark JSON record. + + Overrides metaclass get_main_titles() method. + + Args: + source_record: A JSON object representing a source record. + """ + return [source_record["dct_title_s"]] + + @classmethod + def get_source_record_id(cls, source_record: dict) -> str: + """ + Get source record ID from a JSON record. + + Args: + source_record: A JSON object representing a source record. + """ + return source_record["id"] + + @classmethod + def record_is_deleted(cls, source_record: dict) -> bool: + """ + Determine whether record has a status of deleted. + + ## WIP - defining to enable instantiation of MITAardvark instance. + + Args: + source_record: A JSON object representing a source record. + """ + return False + + def get_optional_fields(self, source_record: dict) -> dict | None: + """ + Retrieve optional TIMDEX fields from a Aardvar JSON record. + + Overrides metaclass get_optional_fields() method. + + Args: + xml: A BeautifulSoup Tag representing a single Datacite record in + oai_datacite XML. + """ + fields: dict = {} + + # alternate_titles + + # content_type + fields["content_type"] = ["Geospatial data"] + + # contributors + + # dates + + # edition + + # format + + # funding_information + + # identifiers + + # languages + fields["languages"] = source_record.get("dct_language_sm") + + # links + + # locations + + # notes + + # publication_information + + # related_items + + # rights + + # subjects + fields["subjects"] = self.get_subjects(source_record) or None + + # summary field + return fields + + @staticmethod + def get_subjects(source_record: dict) -> list[timdex.Subject]: + """Get values from source record for TIMDEX subjects field. + + Unlike other TIMDEX sources, the subject scheme is not known + for each term. The kind here represents the uncontrolled field + in which the term was found. + + DCAT Keyword: https://www.w3.org/TR/vocab-dcat-2/#Property:resource_keyword + DCAT Theme: https://www.w3.org/TR/vocab-dcat-2/#Property:resource_theme + Dublin Core Subject: http://purl.org/dc/terms/subject + + Args: + source_record: A JSON object representing a source record. + """ + subjects = [] + aardvark_subject_fields = { + "dcat_keyword_sm": "DCAT Keyword", + "dcat_theme_sm": "DCAT Theme", + "dct_subject_sm": "Dublin Core Subject", + "gbl_resourceClass_sm": "Subject scheme not provided", + "gbl_resourceType_sm": "Subject scheme not provided", + } + for aardvark_subject_field, kind_value in { + key: value + for key, value in aardvark_subject_fields.items() + if key in source_record + }.items(): + for subject in source_record[aardvark_subject_field]: + subjects.append(timdex.Subject(value=[subject], kind=kind_value)) + return subjects diff --git a/transmogrifier/sources/transformer.py b/transmogrifier/sources/transformer.py index 90eae10..d45e8ab 100644 --- a/transmogrifier/sources/transformer.py +++ b/transmogrifier/sources/transformer.py @@ -8,6 +8,7 @@ from importlib import import_module from typing import Iterator, Optional, TypeAlias, final +import jsonlines from attrs import asdict from bs4 import BeautifulSoup, Tag @@ -29,7 +30,9 @@ class Transformer(ABC): """Base transformer class.""" @final - def __init__(self, source: str, source_records: Iterator[JSON | Tag]) -> None: + def __init__( + self, source: str, source_records: Iterator[dict[str, JSON] | Tag] + ) -> None: """ Initialize Transformer instance. @@ -173,7 +176,9 @@ def get_transformer(cls, source: str) -> type[Transformer]: @final @classmethod - def get_valid_title(cls, source_record_id: str, source_record: Tag | JSON) -> str: + def get_valid_title( + cls, source_record_id: str, source_record: dict[str, JSON] | Tag + ) -> str: """ Retrieves main title(s) from a source record and returns a valid title string. @@ -208,7 +213,7 @@ def get_valid_title(cls, source_record_id: str, source_record: Tag | JSON) -> st @classmethod @abstractmethod - def parse_source_file(cls, source_file: str) -> Iterator[JSON | Tag]: + def parse_source_file(cls, source_file: str) -> Iterator[dict[str, JSON] | Tag]: """ Parse source file and return source records via an iterator. @@ -219,10 +224,46 @@ def parse_source_file(cls, source_file: str) -> Iterator[JSON | Tag]: """ pass + @final + def _transform( + self, source_record: dict[str, JSON] | Tag + ) -> Optional[TimdexRecord]: + """ + Private method called for both XML and JSON transformations, where + all logic is shared except source_record type. + + May not be overridden. + + Args: + source_record: A single source record. + """ + if self.record_is_deleted(source_record): + source_record_id = self.get_source_record_id(source_record) + timdex_record_id = self.get_timdex_record_id( + self.source, source_record_id, source_record + ) + raise DeletedRecord(timdex_record_id) + optional_fields = self.get_optional_fields(source_record) + if optional_fields is None: + return None + else: + fields = { + **self.get_required_fields(source_record), + **optional_fields, + } + + # If citation field was not present, generate citation from other fields + if fields.get("citation") is None: + fields["citation"] = generate_citation(fields) + if fields.get("content_type") is None: + fields["content_type"] = ["Not specified"] + + return TimdexRecord(**fields) + @abstractmethod - def transform(self, source_record: JSON | Tag) -> Optional[TimdexRecord]: + def transform(self, source_record: dict[str, JSON] | Tag) -> Optional[TimdexRecord]: """ - Transform a source record into a TIMDEX record. + Call Transformer._transform method to transform source record to TIMDEX record. Must be overridden by format subclasses. @@ -232,7 +273,7 @@ def transform(self, source_record: JSON | Tag) -> Optional[TimdexRecord]: pass @abstractmethod - def get_required_fields(self, source_record: JSON | Tag) -> dict: + def get_required_fields(self, source_record: dict[str, JSON] | Tag) -> dict: """ Get required TIMDEX fields from a source record. @@ -245,7 +286,7 @@ def get_required_fields(self, source_record: JSON | Tag) -> dict: @classmethod @abstractmethod - def get_main_titles(cls, source_record: JSON | Tag) -> list[str]: + def get_main_titles(cls, source_record: dict[str, JSON] | Tag) -> list[str]: """ Retrieve main title(s) from an source record. @@ -259,7 +300,10 @@ def get_main_titles(cls, source_record: JSON | Tag) -> list[str]: @classmethod @abstractmethod def get_source_link( - cls, source_base_url: str, source_record_id: str, source_record: JSON | Tag + cls, + source_base_url: str, + source_record_id: str, + source_record: dict[str, JSON] | Tag, ) -> str: """ Class method to set the source link for the item. @@ -276,7 +320,7 @@ def get_source_link( @classmethod @abstractmethod def get_timdex_record_id( - cls, source: str, source_record_id: str, source_record: Tag + cls, source: str, source_record_id: str, source_record: dict[str, JSON] | Tag ) -> str: """ Class method to set the TIMDEX record id. @@ -292,7 +336,7 @@ def get_timdex_record_id( @classmethod @abstractmethod - def get_source_record_id(cls, source_record: JSON | Tag) -> str: + def get_source_record_id(cls, source_record: dict[str, JSON] | Tag) -> str: """ Get or generate a source record ID from a source record. @@ -305,7 +349,7 @@ def get_source_record_id(cls, source_record: JSON | Tag) -> str: @classmethod @abstractmethod - def record_is_deleted(cls, source_record: JSON | Tag) -> bool: + def record_is_deleted(cls, source_record: dict[str, JSON] | Tag) -> bool: """ Determine whether record has a status of deleted. @@ -316,7 +360,9 @@ def record_is_deleted(cls, source_record: JSON | Tag) -> bool: """ pass - def get_optional_fields(self, source_record: JSON | Tag) -> Optional[dict]: + def get_optional_fields( + self, source_record: dict[str, JSON] | Tag + ) -> Optional[dict]: """ Retrieve optional TIMDEX fields from a source record. @@ -333,54 +379,35 @@ class JsonTransformer(Transformer): @final @classmethod - def parse_source_file(cls, source_file: str) -> Iterator[JSON]: + def parse_source_file(cls, source_file: str) -> Iterator[dict[str, JSON]]: """ Parse JSON file and return source records as JSON objects via an iterator. May not be overridden. + Validates that records in the file are dicts for proper processing. + Args: source_file: A file containing source records to be transformed. """ - with open(source_file, "rb") as file: - for record in json.load(file): + with jsonlines.open(source_file) as records: + for record in records.iter(type=dict): yield record @final - def transform(self, source_record: JSON) -> Optional[TimdexRecord]: + def transform(self, source_record: dict[str, JSON]) -> Optional[TimdexRecord]: """ - Transform a JSON record into a TIMDEX record. + Call Transformer._transform method to transform JSON record to TIMDEX record. May not be overridden. Args: source_record: A JSON object representing a source record. """ - if self.record_is_deleted(source_record): - source_record_id = self.get_source_record_id(source_record) - timdex_record_id = self.get_timdex_record_id( - self.source, source_record_id, source_record - ) - raise DeletedRecord(timdex_record_id) - optional_fields = self.get_optional_fields(source_record) - if optional_fields is None: - return None - else: - fields = { - **self.get_required_fields(source_record), - **optional_fields, - } - - # If citation field was not present, generate citation from other fields - if fields.get("citation") is None: - fields["citation"] = generate_citation(fields) - if fields.get("content_type") is None: - fields["content_type"] = ["Not specified"] - - return TimdexRecord(**fields) + return self._transform(source_record) @final - def get_required_fields(self, source_record: JSON) -> dict: + def get_required_fields(self, source_record: dict[str, JSON]) -> dict: """ Get required TIMDEX fields from an JSON record. @@ -409,7 +436,7 @@ def get_required_fields(self, source_record: JSON) -> dict: @classmethod @abstractmethod - def get_main_titles(cls, source_record: JSON) -> list[str]: + def get_main_titles(cls, source_record: dict[str, JSON]) -> list[str]: """ Retrieve main title(s) from a JSON record. @@ -422,7 +449,7 @@ def get_main_titles(cls, source_record: JSON) -> list[str]: @classmethod def get_source_link( - cls, source_base_url: str, source_record_id: str, source_record: JSON + cls, source_base_url: str, source_record_id: str, source_record: dict[str, JSON] ) -> str: """ Class method to set the source link for the item. @@ -442,7 +469,7 @@ def get_source_link( @classmethod def get_timdex_record_id( - cls, source: str, source_record_id: str, source_record: JSON + cls, source: str, source_record_id: str, source_record: dict[str, JSON] ) -> str: """ Class method to set the TIMDEX record id. @@ -462,7 +489,7 @@ def get_timdex_record_id( @classmethod @abstractmethod - def get_source_record_id(cls, source_record: JSON) -> str: + def get_source_record_id(cls, source_record: dict[str, JSON]) -> str: """ Get or generate a source record ID from a JSON record. @@ -475,7 +502,7 @@ def get_source_record_id(cls, source_record: JSON) -> str: @classmethod @abstractmethod - def record_is_deleted(cls, source_record: JSON) -> bool: + def record_is_deleted(cls, source_record: dict[str, JSON]) -> bool: """ Determine whether record has a status of deleted. @@ -486,7 +513,7 @@ def record_is_deleted(cls, source_record: JSON) -> bool: """ pass - def get_optional_fields(self, source_record: JSON) -> Optional[dict]: + def get_optional_fields(self, source_record: dict[str, JSON]) -> Optional[dict]: """ Retrieve optional TIMDEX fields from a JSON record. @@ -527,35 +554,14 @@ def parse_source_file(cls, source_file: str) -> Iterator[Tag]: @final def transform(self, source_record: Tag) -> Optional[TimdexRecord]: """ - Transform an XML record into a TIMDEX record. + Call Transformer._transform method to transform XML record to TIMDEX record. May not be overridden. Args: source_record: A BeautifulSoup Tag representing a single XML record. """ - if self.record_is_deleted(source_record): - source_record_id = self.get_source_record_id(source_record) - timdex_record_id = self.get_timdex_record_id( - self.source, source_record_id, source_record - ) - raise DeletedRecord(timdex_record_id) - optional_fields = self.get_optional_fields(source_record) - if optional_fields is None: - return None - else: - fields = { - **self.get_required_fields(source_record), - **optional_fields, - } - - # If citation field was not present, generate citation from other fields - if fields.get("citation") is None: - fields["citation"] = generate_citation(fields) - if fields.get("content_type") is None: - fields["content_type"] = ["Not specified"] - - return TimdexRecord(**fields) + return self._transform(source_record) @final def get_required_fields(self, source_record: Tag) -> dict: diff --git a/transmogrifier/sources/xml/__init__.py b/transmogrifier/sources/xml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transmogrifier/sources/datacite.py b/transmogrifier/sources/xml/datacite.py similarity index 100% rename from transmogrifier/sources/datacite.py rename to transmogrifier/sources/xml/datacite.py diff --git a/transmogrifier/sources/dspace_dim.py b/transmogrifier/sources/xml/dspace_dim.py similarity index 100% rename from transmogrifier/sources/dspace_dim.py rename to transmogrifier/sources/xml/dspace_dim.py diff --git a/transmogrifier/sources/dspace_mets.py b/transmogrifier/sources/xml/dspace_mets.py similarity index 100% rename from transmogrifier/sources/dspace_mets.py rename to transmogrifier/sources/xml/dspace_mets.py diff --git a/transmogrifier/sources/ead.py b/transmogrifier/sources/xml/ead.py similarity index 100% rename from transmogrifier/sources/ead.py rename to transmogrifier/sources/xml/ead.py diff --git a/transmogrifier/sources/marc.py b/transmogrifier/sources/xml/marc.py similarity index 100% rename from transmogrifier/sources/marc.py rename to transmogrifier/sources/xml/marc.py diff --git a/transmogrifier/sources/oaidc.py b/transmogrifier/sources/xml/oaidc.py similarity index 100% rename from transmogrifier/sources/oaidc.py rename to transmogrifier/sources/xml/oaidc.py diff --git a/transmogrifier/sources/springshare.py b/transmogrifier/sources/xml/springshare.py similarity index 98% rename from transmogrifier/sources/springshare.py rename to transmogrifier/sources/xml/springshare.py index 9770e04..96a4fd0 100644 --- a/transmogrifier/sources/springshare.py +++ b/transmogrifier/sources/xml/springshare.py @@ -7,7 +7,7 @@ import transmogrifier.models as timdex from transmogrifier.helpers import validate_date -from transmogrifier.sources.oaidc import OaiDc +from transmogrifier.sources.xml.oaidc import OaiDc logger = logging.getLogger(__name__) diff --git a/transmogrifier/sources/whoas.py b/transmogrifier/sources/xml/whoas.py similarity index 95% rename from transmogrifier/sources/whoas.py rename to transmogrifier/sources/xml/whoas.py index 52e32b4..fecbf91 100644 --- a/transmogrifier/sources/whoas.py +++ b/transmogrifier/sources/xml/whoas.py @@ -1,6 +1,6 @@ from bs4 import Tag -from transmogrifier.sources.dspace_dim import DspaceDim +from transmogrifier.sources.xml.dspace_dim import DspaceDim INVALID_CONTENT_TYPES = [ "article", diff --git a/transmogrifier/sources/zenodo.py b/transmogrifier/sources/xml/zenodo.py similarity index 95% rename from transmogrifier/sources/zenodo.py rename to transmogrifier/sources/xml/zenodo.py index d11eeea..285ed7c 100644 --- a/transmogrifier/sources/zenodo.py +++ b/transmogrifier/sources/xml/zenodo.py @@ -2,7 +2,7 @@ from bs4 import Tag -from transmogrifier.sources.datacite import Datacite +from transmogrifier.sources.xml.datacite import Datacite VALID_CONTENT_TYPES = [ "dataset",