From ed78b253ffb3173c2efd5b5444b06e7ee9ac0a4d Mon Sep 17 00:00:00 2001
From: Paul Makles <me@insrt.uk>
Date: Tue, 1 Oct 2024 21:11:08 +0100
Subject: [PATCH] feat(services/january): website embed generation

---
 Cargo.lock                                   | 347 ++++++++++++++++++-
 crates/core/models/src/v0/embeds.rs          |  35 +-
 crates/services/january/Cargo.toml           |   6 +
 crates/services/january/src/api.rs           |  10 +-
 crates/services/january/src/main.rs          |   1 +
 crates/services/january/src/requests.rs      | 188 +++++++---
 crates/services/january/src/website_embed.rs | 304 ++++++++++++++++
 7 files changed, 814 insertions(+), 77 deletions(-)
 create mode 100644 crates/services/january/src/website_embed.rs

diff --git a/Cargo.lock b/Cargo.lock
index 742393001..63f918891 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -111,6 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
+ "getrandom",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -324,9 +325,9 @@ dependencies = [
 
 [[package]]
 name = "async-recursion"
-version = "1.0.4"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
+checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote 1.0.37",
@@ -1646,6 +1647,29 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "cssparser"
+version = "0.31.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be"
+dependencies = [
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa",
+ "phf 0.11.2",
+ "smallvec",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
+dependencies = [
+ "quote 1.0.37",
+ "syn 2.0.76",
+]
+
 [[package]]
 name = "ct-codecs"
 version = "1.1.1"
@@ -1904,6 +1928,17 @@ dependencies = [
  "syn 1.0.107",
 ]
 
+[[package]]
+name = "derive_more"
+version = "0.99.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
+dependencies = [
+ "proc-macro2",
+ "quote 1.0.37",
+ "syn 2.0.76",
+]
+
 [[package]]
 name = "devise"
 version = "0.3.1"
@@ -1970,6 +2005,21 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
 
+[[package]]
+name = "dtoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
+
+[[package]]
+name = "dtoa-short"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
+dependencies = [
+ "dtoa",
+]
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.5"
@@ -2016,6 +2066,12 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "ego-tree"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642"
+
 [[package]]
 name = "either"
 version = "1.6.1"
@@ -2055,9 +2111,9 @@ dependencies = [
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.31"
+version = "0.8.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
+checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
 dependencies = [
  "cfg-if",
 ]
@@ -2355,6 +2411,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 
+[[package]]
+name = "futf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
+dependencies = [
+ "mac",
+ "new_debug_unreachable",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.21"
@@ -2476,6 +2542,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "generator"
 version = "0.7.0"
@@ -2508,6 +2583,15 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.14"
@@ -2811,6 +2895,20 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "html5ever"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever",
+ "proc-macro2",
+ "quote 1.0.37",
+ "syn 2.0.76",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3735,12 +3833,32 @@ version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd"
 
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+
 [[package]]
 name = "maplit"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
+[[package]]
+name = "markup5ever"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45"
+dependencies = [
+ "log",
+ "phf 0.11.2",
+ "phf_codegen 0.11.2",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
 [[package]]
 name = "match_cfg"
 version = "0.1.0"
@@ -3789,9 +3907,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
 name = "metrics"
@@ -4548,6 +4666,96 @@ dependencies = [
  "sha-1 0.8.2",
 ]
 
+[[package]]
+name = "phf"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
+dependencies = [
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+dependencies = [
+ "phf_macros",
+ "phf_shared 0.11.2",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a"
+dependencies = [
+ "phf_generator 0.11.2",
+ "phf_shared 0.11.2",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
+dependencies = [
+ "phf_shared 0.10.0",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
+dependencies = [
+ "phf_shared 0.11.2",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
+dependencies = [
+ "phf_generator 0.11.2",
+ "phf_shared 0.11.2",
+ "proc-macro2",
+ "quote 1.0.37",
+ "syn 2.0.76",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.0.10"
@@ -4670,6 +4878,12 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
 
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
 [[package]]
 name = "pretty_env_logger"
 version = "0.4.0"
@@ -5159,14 +5373,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.4"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
+checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.3.7",
- "regex-syntax 0.7.5",
+ "regex-automata 0.4.8",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -5180,13 +5394,13 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
+checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.5",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -5203,9 +5417,9 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.5"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "reqwest"
@@ -5498,16 +5712,20 @@ dependencies = [
 name = "revolt-january"
 version = "0.7.14"
 dependencies = [
+ "async-recursion",
  "axum",
  "axum-extra",
+ "encoding_rs",
  "lazy_static",
  "mime",
  "moka",
+ "regex",
  "reqwest 0.12.4",
  "revolt-config",
  "revolt-files",
  "revolt-models",
  "revolt-result",
+ "scraper",
  "serde",
  "serde_json",
  "tempfile",
@@ -6142,6 +6360,22 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scraper"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b90460b31bfe1fc07be8262e42c665ad97118d4585869de9345a84d501a9eaf0"
+dependencies = [
+ "ahash 0.8.11",
+ "cssparser",
+ "ego-tree",
+ "getopts",
+ "html5ever",
+ "once_cell",
+ "selectors",
+ "tendril",
+]
+
 [[package]]
 name = "sct"
 version = "0.7.0"
@@ -6206,6 +6440,25 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "selectors"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
+dependencies = [
+ "bitflags 2.6.0",
+ "cssparser",
+ "derive_more",
+ "fxhash",
+ "log",
+ "new_debug_unreachable",
+ "phf 0.10.1",
+ "phf_codegen 0.10.0",
+ "precomputed-hash",
+ "servo_arc",
+ "smallvec",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -6446,6 +6699,15 @@ dependencies = [
  "syn 1.0.107",
 ]
 
+[[package]]
+name = "servo_arc"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44"
+dependencies = [
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.8.2"
@@ -6562,6 +6824,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
 
+[[package]]
+name = "siphasher"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+
 [[package]]
 name = "slab"
 version = "0.4.6"
@@ -6639,6 +6907,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "state"
 version = "0.5.3"
@@ -6648,6 +6922,32 @@ dependencies = [
  "loom",
 ]
 
+[[package]]
+name = "string_cache"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
+dependencies = [
+ "new_debug_unreachable",
+ "once_cell",
+ "parking_lot",
+ "phf_shared 0.10.0",
+ "precomputed-hash",
+ "serde",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+ "proc-macro2",
+ "quote 1.0.37",
+]
+
 [[package]]
 name = "stringprep"
 version = "0.1.2"
@@ -6845,6 +7145,17 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "tendril"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
+dependencies = [
+ "futf",
+ "mac",
+ "utf-8",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -7437,6 +7748,12 @@ version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
+
 [[package]]
 name = "unicode-xid"
 version = "0.0.4"
diff --git a/crates/core/models/src/v0/embeds.rs b/crates/core/models/src/v0/embeds.rs
index 6649581a0..4ea4d5cfe 100644
--- a/crates/core/models/src/v0/embeds.rs
+++ b/crates/core/models/src/v0/embeds.rs
@@ -119,8 +119,6 @@ auto_derived!(
         #[serde(skip_serializing_if = "Option::is_none")]
         pub video: Option<Video>,
 
-        // #[serde(skip_serializing_if = "Option::is_none")]
-        // opengraph_type: Option<String>,
         /// Site name
         #[serde(skip_serializing_if = "Option::is_none")]
         pub site_name: Option<String>,
@@ -156,11 +154,44 @@ auto_derived!(
 
     /// Embed
     #[serde(tag = "type")]
+    #[derive(Default)]
     pub enum Embed {
         Website(WebsiteMetadata),
         Image(Image),
         Video(Video),
         Text(Text),
+        #[default]
         None,
     }
 );
+
+impl WebsiteMetadata {
+    /// Truncate strings in metadata
+    pub fn truncate(&mut self) {
+        if let Some(s) = self.title.as_mut() {
+            s.truncate(100);
+        }
+
+        if let Some(s) = self.description.as_mut() {
+            s.truncate(1000);
+        }
+
+        if let Some(s) = self.site_name.as_mut() {
+            s.truncate(32);
+        }
+
+        if let Some(s) = self.colour.as_mut() {
+            s.truncate(32);
+        }
+    }
+
+    /// Check if this is considered "empty"
+    pub fn is_empty(&self) -> bool {
+        (self.title.is_none() || self.title.as_ref().is_some_and(|f| f.is_empty()))
+            && (self.description.is_none()
+                || self.description.as_ref().is_some_and(|f| f.is_empty()))
+            && self.special.is_none()
+            && self.video.is_none()
+            && self.image.is_none()
+    }
+}
diff --git a/crates/services/january/Cargo.toml b/crates/services/january/Cargo.toml
index 68f38ab5b..2ceaf8497 100644
--- a/crates/services/january/Cargo.toml
+++ b/crates/services/january/Cargo.toml
@@ -6,15 +6,21 @@ edition = "2021"
 [dependencies]
 # Utility
 mime = "0.3.17"
+regex = "1.11.0"
 tempfile = "3.13.0"
 lazy_static = "1.5.0"
 moka = { version = "0.12.8", features = ["future"] }
 
+# Web scraping
+scraper = "0.20.0"
+encoding_rs = "0.8.34"
+
 # Serialisation
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.68"
 
 # Async runtime
+async-recursion = "1.1.1"
 tokio = { version = "1.0", features = ["full"] }
 
 # Web requests
diff --git a/crates/services/january/src/api.rs b/crates/services/january/src/api.rs
index 91f515211..b875cbc3f 100644
--- a/crates/services/january/src/api.rs
+++ b/crates/services/january/src/api.rs
@@ -1,15 +1,9 @@
 use axum::{extract::Query, response::IntoResponse, routing::get, Json, Router};
 use reqwest::header;
-use revolt_models::v0::Embed;
 use revolt_result::Result;
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
-use axum_extra::{
-    headers::{authorization::Bearer, Authorization},
-    TypedHeader,
-};
-
 use crate::requests::Request;
 
 pub static CACHE_CONTROL: &str = "public, max-age=600, immutable";
@@ -92,6 +86,6 @@ async fn proxy(Query(UrlQuery { url }): Query<UrlQuery>) -> Result<impl IntoResp
 async fn embed(
     Query(UrlQuery { url }): Query<UrlQuery>,
     // TypedHeader(Authorization(_bearer)): TypedHeader<Authorization<Bearer>>,
-) -> Result<Json<Embed>> {
-    Request::generate_embed(&url).await.map(Json)
+) -> Result<impl IntoResponse> {
+    Request::generate_embed(url).await.map(Json)
 }
diff --git a/crates/services/january/src/main.rs b/crates/services/january/src/main.rs
index fcf455ef8..2175ca0af 100644
--- a/crates/services/january/src/main.rs
+++ b/crates/services/january/src/main.rs
@@ -11,6 +11,7 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable};
 
 mod api;
 pub mod requests;
+pub mod website_embed;
 
 #[tokio::main]
 async fn main() -> Result<(), std::io::Error> {
diff --git a/crates/services/january/src/requests.rs b/crates/services/january/src/requests.rs
index b778a3380..c8e540deb 100644
--- a/crates/services/january/src/requests.rs
+++ b/crates/services/january/src/requests.rs
@@ -1,19 +1,23 @@
-use std::{
-    io::{Cursor, Write},
-    time::Duration,
-};
-
+use encoding_rs::{Encoding, UTF_8_INIT};
 use lazy_static::lazy_static;
 use mime::Mime;
-use reqwest::{header::CONTENT_TYPE, redirect, Client, Response};
+use regex::Regex;
+use reqwest::{
+    header::{self, CONTENT_TYPE},
+    redirect, Client, Response,
+};
 use revolt_config::report_internal_error;
 use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size};
 use revolt_models::v0::{Embed, Image, ImageSize, Video};
 use revolt_result::{create_error, Result};
+use std::{
+    io::{Cursor, Write},
+    time::Duration,
+};
 
 lazy_static! {
+    /// Request client
     static ref CLIENT: Client = reqwest::Client::builder()
-        .user_agent("Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)")
         .timeout(Duration::from_secs(10)) // TODO config
         .connect_timeout(Duration::from_secs(5)) // TODO config
         .redirect(redirect::Policy::custom(|attempt| {
@@ -28,6 +32,9 @@ lazy_static! {
         .build()
         .expect("reqwest Client");
 
+    /// Spoof User Agent as Discord
+    static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x).com").expect("valid regex");
+
     /// Cache for proxy results
     static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
         .max_capacity(10_000) // TODO config
@@ -35,7 +42,7 @@ lazy_static! {
         .build();
 
     /// Cache for embed results
-    static ref EMBED_CACHE: moka::future::Cache<String, Result<Embed>> = moka::future::Cache::builder()
+    static ref EMBED_CACHE: moka::future::Cache<String, Embed> = moka::future::Cache::builder()
         .max_capacity(1_000) // TODO config
         .time_to_live(Duration::from_secs(60)) // TODO config
         .build();
@@ -98,57 +105,126 @@ impl Request {
         }
     }
 
-    /// Generate embed for a given URL
-    pub async fn generate_embed(url: &str) -> Result<Embed> {
+    /// Fetch metadata for an image
+    pub async fn fetch_image_metadata(
+        url: &str,
+        request: Option<Request>,
+    ) -> Result<Option<Image>> {
         if let Some(hit) = EMBED_CACHE.get(url).await {
-            hit
+            match hit {
+                Embed::Image(img) => Ok(Some(img)),
+                _ => Ok(None),
+            }
         } else {
-            let Request { response, mime } = Request::new(url).await?;
+            let response = if let Some(Request { response, .. }) = request {
+                response
+            } else {
+                let Request { response, mime } = Request::new(url).await?;
+                if matches!(mime.type_(), mime::IMAGE) {
+                    response
+                } else {
+                    return Err(create_error!(LabelMe));
+                }
+            };
+
+            if let Some((width, height)) =
+                image_size_vec(&report_internal_error!(response.bytes().await)?)
+            {
+                Ok(Some(Image {
+                    url: url.to_owned(),
+                    width,
+                    height,
+                    size: ImageSize::Large,
+                }))
+            } else {
+                Ok(None)
+            }
+        }
+    }
 
-            match (mime.type_(), mime.subtype()) {
+    /// Fetch metadata for an video
+    pub async fn fetch_video_metadata(
+        url: &str,
+        request: Option<Request>,
+    ) -> Result<Option<Video>> {
+        if let Some(hit) = EMBED_CACHE.get(url).await {
+            match hit {
+                Embed::Video(vid) => Ok(Some(vid)),
+                _ => Ok(None),
+            }
+        } else {
+            let response = if let Some(Request { response, .. }) = request {
+                response
+            } else {
+                let Request { response, mime } = Request::new(&url).await?;
+                if matches!(mime.type_(), mime::VIDEO) {
+                    response
+                } else {
+                    return Err(create_error!(LabelMe));
+                }
+            };
+
+            let mut file = report_internal_error!(tempfile::NamedTempFile::new())?;
+            report_internal_error!(
+                file.write_all(&report_internal_error!(response.bytes().await)?)
+            )?;
+
+            if let Some((width, height)) = video_size(&file) {
+                Ok(Some(Video {
+                    url: url.to_owned(),
+                    width: width as usize,
+                    height: height as usize,
+                }))
+            } else {
+                Ok(None)
+            }
+        }
+    }
+
+    /// Generate embed for a given URL
+    pub async fn generate_embed(url: String) -> Result<Embed> {
+        if let Some(hit) = EMBED_CACHE.get(&url).await {
+            Ok(hit)
+        } else {
+            let request = Request::new(&url).await?;
+            let embed = match (request.mime.type_(), request.mime.subtype()) {
                 (_, mime::HTML) => {
-                    // let mut metadata = Metadata::from(resp, url).await?;
-                    // metadata.resolve_external().await;
+                    let content_type = request
+                        .response
+                        .headers()
+                        .get(header::CONTENT_TYPE)
+                        .and_then(|value| value.to_str().ok())
+                        .and_then(|value| value.parse::<Mime>().ok());
 
-                    // if metadata.is_none() {
-                    //     return Ok(Embed::None);
-                    // }
+                    let encoding_name = content_type
+                        .as_ref()
+                        .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
+                        .unwrap_or("utf-8");
 
-                    // Ok(Embed::Website(metadata))
-                    todo!()
-                }
-                (mime::IMAGE, _) => {
-                    if let Some((width, height)) =
-                        image_size_vec(&report_internal_error!(response.bytes().await)?)
-                    {
-                        Ok(Embed::Image(Image {
-                            url: url.to_owned(),
-                            width,
-                            height,
-                            size: ImageSize::Large,
-                        }))
-                    } else {
-                        Ok(Embed::None)
-                    }
-                }
-                (mime::VIDEO, _) => {
-                    let mut file = report_internal_error!(tempfile::NamedTempFile::new())?;
-                    report_internal_error!(
-                        file.write_all(&report_internal_error!(response.bytes().await)?)
-                    )?;
-
-                    if let Some((width, height)) = video_size(&file) {
-                        Ok(Embed::Video(Video {
-                            url: url.to_owned(),
-                            width: width as usize,
-                            height: height as usize,
-                        }))
-                    } else {
-                        Ok(Embed::None)
-                    }
+                    let encoding =
+                        Encoding::for_label(encoding_name.as_bytes()).unwrap_or(&UTF_8_INIT);
+
+                    let bytes = report_internal_error!(request.response.bytes().await)?;
+                    let (text, _, _) = encoding.decode(&bytes);
+
+                    crate::website_embed::create_website_embed(&url, &text)
+                        .await
+                        .map(Embed::Website)
+                        .unwrap_or_default()
                 }
-                _ => Ok(Embed::None),
-            }
+                (mime::IMAGE, _) => Request::fetch_image_metadata(&url, Some(request))
+                    .await
+                    .map(|res| res.map(Embed::Image).unwrap_or_default())
+                    .unwrap_or_default(),
+                (mime::VIDEO, _) => Request::fetch_video_metadata(&url, Some(request))
+                    .await
+                    .map(|res| res.map(Embed::Video).unwrap_or_default())
+                    .unwrap_or_default(),
+                _ => Embed::None,
+            };
+
+            EMBED_CACHE.insert(url.to_owned(), embed.clone()).await;
+            Ok(embed)
         }
     }
 
@@ -156,6 +232,14 @@ impl Request {
     pub async fn new(url: &str) -> Result<Request> {
         let response = CLIENT
             .get(url)
+            .header(
+                "User-Agent",
+                if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) {
+                    "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
+                } else {
+                    "Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)"
+                },
+            )
             .send()
             .await
             .map_err(|_| create_error!(ProxyError))?;
diff --git a/crates/services/january/src/website_embed.rs b/crates/services/january/src/website_embed.rs
new file mode 100644
index 000000000..794fe9dc3
--- /dev/null
+++ b/crates/services/january/src/website_embed.rs
@@ -0,0 +1,304 @@
+use std::collections::HashMap;
+
+use lazy_static::lazy_static;
+use regex::Regex;
+use revolt_models::v0::{
+    BandcampType, Image, ImageSize, LightspeedType, Special, TwitchType, Video, WebsiteMetadata,
+};
+use scraper::{Html, Selector};
+
+/// Create website metadata from URL and document
+pub async fn create_website_embed(original_url: &str, document: &str) -> Option<WebsiteMetadata> {
+    let document = Html::parse_document(document);
+
+    // create selectors
+    let meta_selector = Selector::parse("meta").ok()?;
+    let link_selector = Selector::parse("link").ok()?;
+
+    // extract meta tags
+    let mut meta = HashMap::new();
+    for el in document.select(&meta_selector) {
+        let node = el.value();
+
+        if let (Some(property), Some(content)) = (
+            node.attr("property").or_else(|| node.attr("name")),
+            node.attr("content"),
+        ) {
+            meta.insert(property.to_string(), content.to_string());
+        }
+    }
+
+    // extract rel links
+    let mut link = HashMap::new();
+    for el in document.select(&link_selector) {
+        let node = el.value();
+
+        if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) {
+            link.insert(property.to_string(), content.to_string());
+        }
+    }
+
+    // build metadata
+    let mut metadata = WebsiteMetadata {
+        title: meta
+            .remove("og:title")
+            .or_else(|| meta.remove("twitter:title"))
+            .or_else(|| meta.remove("title"))
+            .map(|s| s.trim().to_owned()),
+        description: meta
+            .remove("og:description")
+            .or_else(|| meta.remove("twitter:description"))
+            .or_else(|| meta.remove("description"))
+            .map(|s| s.trim().to_owned()),
+        image: meta
+            .remove("og:image")
+            .or_else(|| meta.remove("og:image:secure_url"))
+            .or_else(|| meta.remove("twitter:image"))
+            .or_else(|| meta.remove("twitter:image:src"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut url| {
+                // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                if let Some(ch) = url.chars().next() {
+                    if ch == '/' {
+                        url = format!("{}{}", &original_url.trim_end_matches('/'), url);
+                    }
+                }
+
+                let mut size = ImageSize::Preview;
+                if let Some(card) = meta.remove("twitter:card") {
+                    if &card == "summary_large_image" {
+                        size = ImageSize::Large;
+                    }
+                }
+
+                Image {
+                    url: url.to_owned(),
+                    width: meta
+                        .remove("og:image:width")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    height: meta
+                        .remove("og:image:height")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    size,
+                }
+            }),
+        video: meta
+            .remove("og:video")
+            .or_else(|| meta.remove("og:video:url"))
+            .or_else(|| meta.remove("og:video:secure_url"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut url| {
+                // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                if let Some(ch) = url.chars().next() {
+                    if ch == '/' {
+                        url = format!("{}{}", &original_url.trim_end_matches('/'), url);
+                    }
+                }
+
+                Video {
+                    url: url.to_owned(),
+                    width: meta
+                        .remove("og:video:width")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    height: meta
+                        .remove("og:video:height")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                }
+            }),
+        icon_url: link
+            .remove("apple-touch-icon")
+            .or_else(|| link.remove("icon"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut v| {
+                // If relative URL, prepend root URL.
+                if let Some(ch) = v.chars().next() {
+                    if ch == '/' {
+                        v = format!("{}{}", &original_url.trim_end_matches('/'), v);
+                    }
+                }
+
+                v
+            }),
+        colour: meta.remove("theme-color").map(|s| s.trim().to_owned()),
+        site_name: meta.remove("og:site_name").map(|s| s.trim().to_owned()),
+        url: meta
+            .remove("og:url")
+            .or_else(|| Some(original_url.to_owned())),
+        original_url: Some(original_url.to_owned()),
+        special: None,
+    };
+
+    // populate extra metadata for popular websites
+    populate_special(original_url.to_owned(), &mut metadata);
+
+    // TODO: these do not work because compiler is tripping:
+
+    // fetch video size if missing
+    // if let Some(Video { width, height, url }) = &metadata.video {
+    //     if width == &0 || height == &0 {
+    //         metadata.video = match crate::requests::Request::fetch_video_metadata(url, None).await {
+    //             Ok(Some(video)) => Some(video),
+    //             _ => None,
+    //         }
+    //     }
+    // }
+
+    // fetch image size if missing
+    // if let Some(Image {
+    //     width, height, url, ..
+    // }) = &metadata.image
+    // {
+    //     if width == &0 || height == &0 {
+    //         metadata.image = match crate::requests::Request::fetch_image_metadata(url, None).await {
+    //             Ok(Some(image)) => Some(image),
+    //             _ => None,
+    //         }
+    //     }
+    // }
+
+    // truncate data
+    metadata.truncate();
+
+    // if it's empty, don't return anything
+    if metadata.is_empty() {
+        None
+    } else {
+        Some(metadata)
+    }
+}
+
+pub fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
+    lazy_static! {
+        static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
+
+        static ref RE_LIGHTSPEED: Regex = Regex::new("^(?:https?://)?(?:[\\w]+\\.)?lightspeed\\.tv/([a-z0-9_]{4,25})").unwrap();
+
+        static ref RE_TWITCH: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/([a-z0-9_]+)($|\\?)").unwrap();
+        static ref RE_TWITCH_VOD: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/videos/([0-9]+)($|\\?)").unwrap();
+        static ref RE_TWITCH_CLIP: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/(?:[a-z0-9_]+)/clip/([A-z0-9_-]+)($|\\?)").unwrap();
+
+        static ref RE_SPOTIFY: Regex = Regex::new("^(?:https?://)?open.spotify.com/(track|user|artist|album|playlist)/([A-z0-9]+)").unwrap();
+        static ref RE_SOUNDCLOUD: Regex = Regex::new("^(?:https?://)?soundcloud.com/([a-zA-Z0-9-]+)/([A-z0-9-]+)").unwrap();
+        static ref RE_BANDCAMP: Regex = Regex::new("^(?:https?://)?(?:[A-z0-9_-]+).bandcamp.com/(track|album)/([A-z0-9_-]+)").unwrap();
+        static ref RE_APPLE_MUSIC: Regex = Regex::new("^(?:https?://)?music\\.apple\\.com/(?:[a-z]{2}/)?album/(?:[a-zA-Z0-9-]+)/(\\d+)(?:\\?i=(\\d+))?").unwrap();
+
+        static ref RE_STREAMABLE: Regex = Regex::new("^(?:https?://)?(?:www\\.)?streamable\\.com/([\\w\\d-]+)").unwrap();
+
+        static ref RE_GIF: Regex = Regex::new("^(?:https?://)?(www\\.)?(gifbox\\.me/view|yiffbox\\.me/view|tenor\\.com/view|giphy\\.com/gifs|gfycat\\.com|redgifs\\.com/watch)/[\\w\\d-]+").unwrap();
+    }
+
+    let url = metadata
+        .url
+        .as_ref()
+        .or(metadata.original_url.as_ref())
+        .unwrap_or(&original_url)
+        .as_str();
+
+    metadata.special = if let Some(captures) = RE_STREAMABLE.captures_iter(url).next() {
+        Some(Special::Streamable {
+            id: captures[1].to_string(),
+        })
+    } else if let Some(captures) = RE_YOUTUBE.captures_iter(url).next() {
+        lazy_static! {
+            static ref RE_TIMESTAMP: Regex = Regex::new("(?:\\?|&)(?:t|start)=([\\w]+)").unwrap();
+        }
+
+        if let Some(video) = &metadata.video {
+            if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(&video.url).next() {
+                Some(Special::YouTube {
+                    id: captures[1].to_string(),
+                    timestamp: Some(timestamp_captures[1].to_string()),
+                })
+            } else {
+                Some(Special::YouTube {
+                    id: captures[1].to_string(),
+                    timestamp: None,
+                })
+            }
+        } else {
+            None
+        }
+    } else if let Some(captures) = RE_LIGHTSPEED.captures_iter(url).next() {
+        Some(Special::Lightspeed {
+            id: captures[1].to_string(),
+            content_type: LightspeedType::Channel,
+        })
+    } else if let Some(captures) = RE_TWITCH.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Channel,
+        })
+    } else if let Some(captures) = RE_TWITCH_VOD.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Video,
+        })
+    } else if let Some(captures) = RE_TWITCH_CLIP.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Clip,
+        })
+    } else if let Some(captures) = RE_SPOTIFY.captures_iter(url).next() {
+        Some(Special::Spotify {
+            content_type: captures[1].to_string(),
+            id: captures[2].to_string(),
+        })
+    } else if RE_SOUNDCLOUD.is_match(url) {
+        Some(Special::Soundcloud)
+    } else if RE_BANDCAMP.is_match(url) {
+        lazy_static! {
+            static ref RE_TRACK: Regex = Regex::new("track=(\\d+)").unwrap();
+            static ref RE_ALBUM: Regex = Regex::new("album=(\\d+)").unwrap();
+        }
+
+        if let Some(video) = &metadata.video {
+            if let Some(captures) = RE_TRACK.captures_iter(&video.url).next() {
+                Some(Special::Bandcamp {
+                    content_type: BandcampType::Track,
+                    id: captures[1].to_string(),
+                })
+            } else {
+                RE_ALBUM
+                    .captures_iter(&video.url)
+                    .next()
+                    .map(|captures| Special::Bandcamp {
+                        content_type: BandcampType::Album,
+                        id: captures[1].to_string(),
+                    })
+            }
+        } else {
+            None
+        }
+    } else if RE_GIF.is_match(url) {
+        Some(Special::GIF)
+    } else {
+        RE_APPLE_MUSIC
+            .captures_iter(url)
+            .next()
+            .map(|captures| Special::AppleMusic {
+                album_id: captures[1].to_string(),
+                track_id: captures.get(2).map(|m| m.as_str().to_string()),
+            })
+    };
+
+    // add colours for popular websites
+    if let Some(special) = &metadata.special {
+        match special {
+            Special::YouTube { .. } => metadata.colour = Some("#FF424F".to_string()),
+            Special::Twitch { .. } => metadata.colour = Some("#7B68EE".to_string()),
+            Special::Lightspeed { .. } => metadata.colour = Some("#7445D9".to_string()),
+            Special::Spotify { .. } => metadata.colour = Some("#1ABC9C".to_string()),
+            Special::Soundcloud { .. } => metadata.colour = Some("#FF7F50".to_string()),
+            Special::AppleMusic { .. } => metadata.colour = Some("#FA233B".to_string()),
+            _ => {}
+        }
+    }
+}