Merge pull request #12 from polywrap/fixes

chore: minor fixes | /workflows/cd

.github/workflows/cd.yaml

@@ -31,10 +31,10 @@ jobs:
         run: yarn install --nonInteractive --frozen-lockfile --prefer-offline
 
       - name: Build
-        run: yarn build
+        run: npx polywrap build
 
       - name: Deploy
-        run: yarn deploy
+        run: npx polywrap deploy
         env:
           POLYWRAP_WRAPSCAN_AUTH_HEADER_PROD: ${{secrets.POLYWRAP_WRAPSCAN_AUTH_HEADER_PROD}}
 

.github/workflows/ci.yaml

@@ -0,0 +1,33 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Read .nvmrc
+        run: echo ::set-output name=NVMRC::$(cat .nvmrc)
+        id: nvm
+
+      - name: Setup Node.js
+        uses: actions/setup-node@master
+        with:
+          node-version: '${{ steps.nvm.outputs.NVMRC }}'
+
+      - name: Install dependencies
+        run: yarn install --nonInteractive --frozen-lockfile --prefer-offline
+
+      - name: Build
+        run: npx polywrap build
+
+      - name: Test
+        run: yarn test

README.md

@@ -1,15 +1,72 @@
-# Polywrap Wasm Wrapper Template
-A simple starter template for a Rust Wasm wrapper. For more information on how this project works, and a step by step on how to extend its behavior, see the documentation [here](https://docs.polywrap.io/).
+# Web-Scraper
+A wrap of the popular [scraper](https://crates.io/crates/scraper) crate.
 
-# How To Run
+## Integrate
+### 1. Polywrap Setup
+`polywrap.graphql`:
+```graphql
+#import * into WebScraper from "wrapscan.io/polywrap/web-scraper@1"
+```
 
-## Install Dependencies
-`nvm install && nvm use`  
-`yarn`  
+`polywrap.yaml`:
+```yaml
+format: 0.3.0
+project:
+  name: my-app
+  type: app/typescript|python|rust|kotlin|swift
+source:
+  schema: ./polywrap.graphql
+```
 
-## Codegen & Build
-`yarn codegen`  
-`yarn build`  
+Codegen:
+```bash
+$ polywrap codegen
+```
 
-## Test
-`yarn test`  
+### 2. Run The Wrap
+
+`app/typescript`:
+```typescript
+const webScraper = new WebScraper();
+
+await webScraper.get_text({
+  url: "...",
+});
+```
+
+`app/python`:
+```python
+web_scraper = WebScraper()
+
+result = web_scraper.get_text({
+    "url": "..."
+})
+```
+
+`app/rust`:
+```rust
+let web_scraper = WebScraper::new();
+
+let result = web_scraper.get_text(
+    &WebScraperArgsGetText{
+        url: "..."
+    }
+).unwrap();
+```
+
+`app/kotlin`:
+```kotlin
+val webScraper = WebScraper(client)
+
+val result = webScraper.get_text(
+  WebScraperArgsGetText("...")
+).getOrThrow()
+```
+
+`app/swift`:
+```swift
+let web_scraper = WebScraper()
+try? web_scraper.get_text(
+  args: WebScraperArgsGetText(url: "...")
+)
+```

package.json

@@ -1,16 +1,8 @@
 {
-  "name": "template-wasm-rs",
-  "description": "Polywrap Rust Wrapper Template",
   "private": true,
-  "version": "0.10.6",
   "scripts": {
-    "codegen": "npx polywrap codegen",
-    "build": "npx polywrap build",
-    "deploy": "npx polywrap deploy",
-    "test": "yarn test:e2e && yarn test:workflow",
-    "test:e2e": "yarn test:e2e:codegen && jest --passWithNoTests --runInBand --verbose",
-    "test:e2e:codegen": "npx polywrap codegen -m ./src/__tests__/types/polywrap.app.yaml -g ./src/__tests__/types/wrap",
-    "test:workflow": "npx polywrap test"
+    "test": "yarn test:codegen && jest --passWithNoTests --runInBand --verbose",
+    "test:codegen": "npx polywrap codegen -m ./src/__tests__/types/polywrap.app.yaml -g ./src/__tests__/types/wrap"
   },
   "devDependencies": {
     "@types/jest": "26.0.8",

polywrap.deploy.yaml

@@ -12,7 +12,7 @@ jobs:
         package: http
         uri: $$ipfs_deploy
         config:
-          postUrl: https://wraps.wrapscan.io/r/polywrap/[email protected].0
+          postUrl: https://wraps.wrapscan.io/r/polywrap/[email protected].1
           headers:
             - name: Authorization
               value: $POLYWRAP_WRAPSCAN_AUTH_HEADER_PROD

polywrap.docs.yaml

@@ -0,0 +1,5 @@
+format: 0.1.0
+description: Scrape text and links from website urls
+website: https://polywrap.io
+repository: https://github.com/polywrap/web-scraper
+readme: ./README.md

polywrap.graphql

@@ -2,5 +2,5 @@
 
 type Module {
   get_text(url: String!): String!
-  get_links(uri: String!): String!
+  get_links(url: String!): String!
 }

polywrap.yaml

@@ -1,9 +1,9 @@
-format: 0.3.0
+format: 0.5.0
 project:
   name: web-scraper
   type: wasm/rust
 source:
   module: ./Cargo.toml
   schema: ./polywrap.graphql
 extensions:
-  build: ./polywrap.build.yaml
+  docs: ./polywrap.docs.yaml

src/__tests__/e2e/integration.spec.ts

@@ -1,45 +1,49 @@
 import { PolywrapClient } from "@polywrap/client-js";
-import * as App from "../types/wrap";
-import path from "path";
 
-jest.setTimeout(60000);
+import { WebScraper } from "../types/wrap";
 
-describe("Web Scraper Wrapper End to End Tests", () => {
+jest.setTimeout(60000);
 
-  const client: PolywrapClient = new PolywrapClient();
-  let wrapperUri: string;
+describe("WebScraper", () => {
 
-  beforeAll(() => {
-    const dirname: string = path.resolve(__dirname);
-    const wrapperPath: string = path.join(dirname, "..", "..", "..");
-    wrapperUri = `fs/${wrapperPath}/build`;
-  })
+  const client = new PolywrapClient();
+  const localWrap = `file://${__dirname}/../../../build`
 
-  it("calls get_links", async () => {
-    const uri = "https://polywrap.io";
+  const webScraper = new WebScraper(client, undefined, localWrap);
 
-    const result = await client.invoke<App.Module_GetLinks>({
-      uri: wrapperUri,
-      method: "get_links",
-      args: { uri: uri }
+  it("get_links", async () => {
+    const result = await webScraper.get_links({
+      url: "https://webscraper.io/test-sites/e-commerce/allinone"
     });
-
     expect(result.ok).toBeTruthy();
     if (!result.ok) return;
-    // Add additional assertions as needed
+    expect(result.value).toContain(`/
+#page-top
+/
+/cloud-scraper
+/pricing
+#section3
+/documentation
+/tutorials
+/how-to-videos
+/test-sites
+https://forum.webscraper.io/
+https://chrome.google.com/webstore/detail/web-scraper/jnhgnonknehpejjnehehllkliplmbmhn?hl=en
+https://cloud.webscraper.io/
+/test-sites/e-commerce/allinone
+/test-sites/e-commerce/allinone/phones
+/test-sites/e-commerce/allinone/computers`
+    );
   });
 
-  it("calls get_text", async () => {
-    const url = "https://polywrap.io";
-
-    const result = await client.invoke<App.Module_GetText>({
-      uri: wrapperUri,
-      method: "get_text",
-      args: { url: url }
+  it("get_text", async () => {
+    const result = await webScraper.get_text({
+      url: "\nhttps://webscraper.io/test-sites/e-commerce/allinone\n"
     });
-
     expect(result.ok).toBeTruthy();
     if (!result.ok) return;
-    // Add additional assertions as needed
+    expect(result.value).toContain(
+      `Web Scraper\nCloud Scraper\n`
+    );
   });
 });

src/__tests__/types/polywrap.app.yaml

@@ -1,9 +1,9 @@
-format: 0.3.0
+format: 0.4.0
 project:
   name: sample-typescript-type-generation
   type: app/typescript
 source:
   schema: ./schema.graphql
   import_abis:
-    - uri: "wrap://ens/sample.eth"
+    - uri: "wrap://wrapscan.io/polywrap/web-scraper"
       abi: "../../../build/wrap.info"

src/__tests__/types/schema.graphql

@@ -1 +1 @@
-#import * into Template from "wrap://ens/sample.eth"
+#import * into WebScraper from "wrap://wrapscan.io/polywrap/web-scraper"

src/lib.rs

@@ -15,7 +15,7 @@ fn extract_text(element: &ElementRef) -> String {
 impl ModuleTrait for Module {
     fn get_links(args: ArgsGetLinks) -> Result<String, String> {
         let result = HttpModule::get(&ArgsGet {
-            url: args.uri.clone(),
+            url: args.url.clone(),
             request: Some(HttpRequest{
                 response_type: HttpResponseType::TEXT,
                 headers: None,
@@ -37,7 +37,7 @@ impl ModuleTrait for Module {
             }
         }
 
-        Ok(links.join(", "))
+        Ok(links.join("\n"))
     }
 
     fn get_text(args: ArgsGetText) -> Result<String, String> {
@@ -70,6 +70,6 @@ impl ModuleTrait for Module {
             }
         }
 
-        Ok(text_vec.join(" ").trim().to_string())
+        Ok(text_vec.join("\n"))
     }
 }