Merge pull request #243 from jshemas/aug-9

august clean up
jshemas · Aug 17, 2024 · 5d9874d · 5d9874d
2 parents 80b1800 + 4eda428
commit 5d9874d
Show file tree

Hide file tree

Showing 19 changed files with 104 additions and 68 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Change Log
 
+## 6.8.0
+
+- Updating how `onlyGetOpenGraphInfo` works. By default it is `false` but now it accepts an array of properties for which no fallback should be used.
+- Updating how you get types `import { SuccessResult } from 'open-graph-scraper/types';`. See readme for details.
+- Updating dependencies
+
 ## 6.7.2
 
 - Adding `types` to the npm export. You can now use `import { SuccessResult } from 'open-graph-scraper/types/lib/types';`
@@ -100,7 +106,7 @@
 - Modified the `url` property in `OpenGraphScraperOptions` to be an optional property since you don't need this when using just `html`
 - `Type` can optional in `ImageObject` since type is not set it it's invalid
 - Take all of the `customMetaTags` out of base of `ogObject` and store them into `ogObject.customMetaTags`
-- The interal meta properties can be string arrays
+- The internal meta properties can be string arrays
 - Updating Dependencies
 
 ## 6.1.0
@@ -119,7 +125,7 @@
 - Replace `GOT` with [fetch](https://nodejs.org/docs/latest-v18.x/api/globals.html#fetch)!
 - Only supporting `node18` or higher going forward
 - Updated how options work. `Fetch` and `OGS` options no longer being mixed together, users can now set [fetch options](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) using `options.fetchOptions`
-- Remove any ogImages/ogVideos/twitterImages/twitterPlayers/musicSongs resultes that have no url
+- Remove any ogImages/ogVideos/twitterImages/twitterPlayers/musicSongs results that have no url
 - The `downloadLimit` option has been removed in favor of just using timeouts.
 - Limit ogImages/ogVideos/twitterImages/twitterPlayers/musicSongs to 10 items
 - Adding html to the `SuccessResult` of `OGS`

diff --git a/README.md b/README.md
@@ -67,6 +67,31 @@ Check the return for a ```success``` flag. If success is set to true, then the u
 
 Note: `open-graph-scraper` uses the [Fetch API](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html#fetch) for requests and most of [Fetch's options](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) should work as `open-graph-scraper`'s `fetchOptions` options.
 
+## Types And Import Example
+
+```javascript
+// example of how to get types
+import type { SuccessResult } from 'open-graph-scraper/types';
+const example: SuccessResult = {
+  result: { ogTitle: 'this is a title' },
+  error: false,
+  response: {},
+  html: '<html></html>'
+}
+
+// import example
+import ogs from 'open-graph-scraper';
+const options = { url: 'http://ogp.me/' };
+ogs(options)
+  .then((data) => {
+    const { error, html, result, response } = data;
+    console.log('error:', error);  // This returns true or false. True if there was an error. The error itself is inside the result object.
+    console.log('html:', html); // This contains the HTML of page
+    console.log('result:', result); // This contains all of the Open Graph results
+    console.log('response:', response); // This contains response from the Fetch API
+  });
+```
+
 ## Custom Meta Tag Example
 
 ```javascript
@@ -123,7 +148,7 @@ The request header is set to [undici](https://github.com/nodejs/undici) by defau
 
 ```javascript
 const ogs = require("open-graph-scraper");
-const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36';
+const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36';
 ogs({ url: 'https://www.wikipedia.org/', fetchOptions: { headers: { 'user-agent': userAgent } } })
   .then((data) => {
     const { error, html, result, response } = data;

diff --git a/lib/extract.ts b/lib/extract.ts
@@ -5,7 +5,7 @@ import fields from './fields';
 import mediaSetup from './media';
 import { unescapeScriptText } from './utils';
 
-import type { OgObjectInteral, OpenGraphScraperOptions } from './types';
+import type { OgObjectInternal, OpenGraphScraperOptions } from './types';
 
 /**
  * extract all of the meta tags needed for ogs
@@ -16,7 +16,7 @@ import type { OgObjectInteral, OpenGraphScraperOptions } from './types';
  *
  */
 export default function extractMetaTags(body: string, options: OpenGraphScraperOptions) {
-  let ogObject: OgObjectInteral = { success: true };
+  let ogObject: OgObjectInternal = { success: true };
   const $ = load(body);
   const metaFields = fields;
 

diff --git a/lib/fallback.ts b/lib/fallback.ts
@@ -8,7 +8,7 @@ import {
   isUrlValid,
 } from './utils';
 import type {
-  OpenGraphScraperOptions, ImageObject, OgObjectInteral, OnlyGetOpenGraphInfoItem,
+  OpenGraphScraperOptions, ImageObject, OgObjectInternal, OnlyGetOpenGraphInfoItem,
 } from './types';
 
 const doesElementExist = (selector:string, attribute:string, $: CheerioAPI) => (
@@ -24,7 +24,7 @@ const doesElementExist = (selector:string, attribute:string, $: CheerioAPI) => (
  * @return {object} object with ogs results with updated fallback values
  *
  */
-export function fallback(ogObject: OgObjectInteral, options: OpenGraphScraperOptions, $: CheerioAPI, body: string) {
+export function fallback(ogObject: OgObjectInternal, options: OpenGraphScraperOptions, $: CheerioAPI, body: string) {
   const shouldFallback = (key: OnlyGetOpenGraphInfoItem): boolean => {
     if (!options.onlyGetOpenGraphInfo) {
       return true;

diff --git a/lib/fields.ts b/lib/fields.ts
@@ -1,9 +1,9 @@
-import type { OgObjectInteral } from './types';
+import type { OgObjectInternal } from './types';
 
 type Fields = {
   multiple: boolean;
   property: string;
-  fieldName: keyof OgObjectInteral;
+  fieldName: keyof OgObjectInternal;
 }[];
 
 /**

diff --git a/lib/media.ts b/lib/media.ts
@@ -3,7 +3,7 @@ import { removeNestedUndefinedValues } from './utils';
 import type {
   ImageObject,
   MusicSongObject,
-  OgObjectInteral,
+  OgObjectInternal,
   TwitterImageObject,
   TwitterPlayerObject,
   VideoObject,
@@ -84,7 +84,7 @@ const zip = (array: any, ...args: any) => {
  * @return {object} object with ogs results with updated media values
  *
  */
-export function mediaSetup(ogObject: OgObjectInteral) {
+export function mediaSetup(ogObject: OgObjectInternal) {
   // sets ogImage property/width/height/type to empty array if one these exists
   if (
     ogObject.ogImageSecureURL

diff --git a/lib/types.ts b/lib/types.ts
@@ -115,7 +115,7 @@ export interface MusicSongObject {
   url: string;
 }
 
-export interface OgObjectInteral {
+export interface OgObjectInternal {
   alAndroidAppName?: string;
   alAndroidClass?: string;
   alAndroidPackage?: string;
@@ -319,7 +319,7 @@ export interface OgObjectInteral {
 
 // Omit values from mediaMapperProperties
 export type OgObject = Omit<
-OgObjectInteral,
+OgObjectInternal,
 'musicSongDisc' |
 'musicSongProperty' |
 'musicSongTrack' |

diff --git a/lib/utils.ts b/lib/utils.ts
@@ -1,7 +1,7 @@
 import isUrl from './isUrl';
 import type {
   CustomMetaTags,
-  OgObjectInteral,
+  OgObjectInternal,
   OpenGraphScraperOptions,
   ValidatorSettings,
 } from './types';
@@ -93,13 +93,13 @@ export function isThisANonHTMLUrl(url: string): boolean {
 }
 
 /**
- * Find and delete nested undefs
+ * Find and delete nested undefineds
  *
  * @param {object} object - object to be cleaned
- * @return {object} object without nested undefs
+ * @return {object} object without nested undefineds
  *
  */
-export function removeNestedUndefinedValues(object: Record<string, any>): OgObjectInteral {
+export function removeNestedUndefinedValues(object: Record<string, any>): OgObjectInternal {
   Object.entries(object).forEach(([key, value]) => {
     if (value && typeof value === 'object') removeNestedUndefinedValues(value);
     else if (value === undefined) delete object[key];

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "open-graph-scraper",
   "description": "Node.js scraper module for Open Graph and Twitter Card info",
-  "version": "6.7.2",
+  "version": "6.8.0",
   "license": "MIT",
   "main": "./dist/cjs/index.js",
   "types": "./types/index.d.ts",
@@ -11,7 +11,7 @@
       "import": "./dist/esm/index.js",
       "require": "./dist/cjs/index.js"
     },
-    "./types/lib/types": "./types/lib/types.d.ts"
+    "./types": "./types/lib/types.d.ts"
   },
   "scripts": {
     "build:cjs": "tsc --project tsconfig.build.json --module commonjs --outDir dist/cjs/",
@@ -39,7 +39,7 @@
     "chardet": "^2.0.0",
     "cheerio": "^1.0.0-rc.12",
     "iconv-lite": "^0.6.3",
-    "undici": "^6.19.5"
+    "undici": "^6.19.7"
   },
   "files": [
     "/dist",
@@ -48,9 +48,9 @@
     "CHANGELOG.md"
   ],
   "devDependencies": {
-    "@snyk/protect": "^1.1292.2",
+    "@snyk/protect": "^1.1292.4",
     "@types/mocha": "^10.0.7",
-    "@types/node": "^18.19.42",
+    "@types/node": "^18.19.44",
     "@typescript-eslint/eslint-plugin": "^7.18.0",
     "@typescript-eslint/parser": "^7.18.0",
     "chai": "^4.5.0",
@@ -59,8 +59,8 @@
     "eslint-config-airbnb-typescript": "^18.0.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-mocha": "^10.5.0",
-    "eslint-plugin-promise": "^7.0.0",
-    "mocha": "^10.6.0",
+    "eslint-plugin-promise": "^7.1.0",
+    "mocha": "^10.7.3",
     "nyc": "^17.0.0",
     "sinon": "^18.0.0",
     "ts-mocha": "^10.0.0",

diff --git a/tests/integration/video.spec.ts b/tests/integration/video.spec.ts
@@ -2,9 +2,11 @@ import { expect } from 'chai';
 
 import ogs from '../../index';
 
-describe('video', function () {
+// TODO: youtube is blocking requests from github, will need to find a way around this
+describe.skip('video', function () {
   it('Test Youtube Video - Should Return correct Open Graph Info', function () {
-    return ogs({ url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' }).then(function ({ error, result, response }) {
+    const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36';
+    return ogs({ url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', fetchOptions: { headers: { 'user-agent': userAgent } } }).then(function ({ error, result, response }) {
       console.log('error:', error);
       console.log('result:', result);
       expect(error).to.be.eql(false);
@@ -111,7 +113,8 @@ describe('video', function () {
   });
 
   it('Test Youtube Video with bad escape sequence - Should Return correct Open Graph Info', function () {
-    return ogs({ url: 'https://www.youtube.com/watch?v=nFbKMg4E3JM' }).then(function ({ error, result, response }) {
+    const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36';
+    return ogs({ url: 'https://www.youtube.com/watch?v=nFbKMg4E3JM', fetchOptions: { headers: { 'user-agent': userAgent } } }).then(function ({ error, result, response }) {
       console.log('error:', error);
       console.log('result:', result);
       expect(error).to.be.eql(false);

diff --git a/types/index.d.ts b/types/index.d.ts
@@ -4,7 +4,8 @@ import type { ErrorResult, OpenGraphScraperOptions, SuccessResult } from './lib/
  * for scraping Open Graph and Twitter Card info off a website.
  *
  * @param {object} options - The options used by Open Graph Scraper
- * @param {boolean} [options.onlyGetOpenGraphInfo] - Only fetch open graph info and don't fall back on anything else.
+ * @param {boolean|string[]} [options.onlyGetOpenGraphInfo] - Only fetch open graph info and don't fall back on
+ * anything else.
  * @param {object} [options.customMetaTags] - Here you can define custom meta tags you want to scrape.
  * @param {object} [options.fetchOptions] - Sets the options used by fetch for the http requests
  * @param {object} [options.urlValidatorSettings] - Sets the options used by validator.js for testing the URL

diff --git a/types/lib/extract.d.ts b/types/lib/extract.d.ts
@@ -1,4 +1,4 @@
-import type { OgObjectInteral, OpenGraphScraperOptions } from './types';
+import type { OgObjectInternal, OpenGraphScraperOptions } from './types';
 /**
  * extract all of the meta tags needed for ogs
  *
@@ -7,4 +7,4 @@ import type { OgObjectInteral, OpenGraphScraperOptions } from './types';
  * @return {object} object with ogs results
  *
  */
-export default function extractMetaTags(body: string, options: OpenGraphScraperOptions): OgObjectInteral;
+export default function extractMetaTags(body: string, options: OpenGraphScraperOptions): OgObjectInternal;
diff --git a/types/lib/fallback.d.ts b/types/lib/fallback.d.ts
@@ -1,5 +1,5 @@
 import type { CheerioAPI } from 'cheerio';
-import type { OpenGraphScraperOptions, OgObjectInteral } from './types';
+import type { OpenGraphScraperOptions, OgObjectInternal } from './types';
 /**
  * ogs fallbacks
  *
@@ -9,5 +9,5 @@ import type { OpenGraphScraperOptions, OgObjectInteral } from './types';
  * @return {object} object with ogs results with updated fallback values
  *
  */
-export declare function fallback(ogObject: OgObjectInteral, options: OpenGraphScraperOptions, $: CheerioAPI, body: string): OgObjectInteral;
+export declare function fallback(ogObject: OgObjectInternal, options: OpenGraphScraperOptions, $: CheerioAPI, body: string): OgObjectInternal;
 export default fallback;