-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #12 from polywrap/fixes
chore: minor fixes | /workflows/cd
- Loading branch information
Showing
17 changed files
with
774 additions
and
708 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
name: ci | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
pull_request: | ||
|
||
jobs: | ||
ci: | ||
runs-on: ubuntu-latest | ||
timeout-minutes: 60 | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v2 | ||
|
||
- name: Read .nvmrc | ||
run: echo ::set-output name=NVMRC::$(cat .nvmrc) | ||
id: nvm | ||
|
||
- name: Setup Node.js | ||
uses: actions/setup-node@master | ||
with: | ||
node-version: '${{ steps.nvm.outputs.NVMRC }}' | ||
|
||
- name: Install dependencies | ||
run: yarn install --nonInteractive --frozen-lockfile --prefer-offline | ||
|
||
- name: Build | ||
run: npx polywrap build | ||
|
||
- name: Test | ||
run: yarn test |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,15 +1,72 @@ | ||
# Polywrap Wasm Wrapper Template | ||
A simple starter template for a Rust Wasm wrapper. For more information on how this project works, and a step by step on how to extend its behavior, see the documentation [here](https://docs.polywrap.io/). | ||
# Web-Scraper | ||
A wrap of the popular [scraper](https://crates.io/crates/scraper) crate. | ||
|
||
# How To Run | ||
## Integrate | ||
### 1. Polywrap Setup | ||
`polywrap.graphql`: | ||
```graphql | ||
#import * into WebScraper from "wrapscan.io/polywrap/web-scraper@1" | ||
``` | ||
|
||
## Install Dependencies | ||
`nvm install && nvm use` | ||
`yarn` | ||
`polywrap.yaml`: | ||
```yaml | ||
format: 0.3.0 | ||
project: | ||
name: my-app | ||
type: app/typescript|python|rust|kotlin|swift | ||
source: | ||
schema: ./polywrap.graphql | ||
``` | ||
## Codegen & Build | ||
`yarn codegen` | ||
`yarn build` | ||
Codegen: | ||
```bash | ||
$ polywrap codegen | ||
``` | ||
|
||
## Test | ||
`yarn test` | ||
### 2. Run The Wrap | ||
|
||
`app/typescript`: | ||
```typescript | ||
const webScraper = new WebScraper(); | ||
|
||
await webScraper.get_text({ | ||
url: "...", | ||
}); | ||
``` | ||
|
||
`app/python`: | ||
```python | ||
web_scraper = WebScraper() | ||
|
||
result = web_scraper.get_text({ | ||
"url": "..." | ||
}) | ||
``` | ||
|
||
`app/rust`: | ||
```rust | ||
let web_scraper = WebScraper::new(); | ||
|
||
let result = web_scraper.get_text( | ||
&WebScraperArgsGetText{ | ||
url: "..." | ||
} | ||
).unwrap(); | ||
``` | ||
|
||
`app/kotlin`: | ||
```kotlin | ||
val webScraper = WebScraper(client) | ||
|
||
val result = webScraper.get_text( | ||
WebScraperArgsGetText("...") | ||
).getOrThrow() | ||
``` | ||
|
||
`app/swift`: | ||
```swift | ||
let web_scraper = WebScraper() | ||
try? web_scraper.get_text( | ||
args: WebScraperArgsGetText(url: "...") | ||
) | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,7 +12,7 @@ jobs: | |
package: http | ||
uri: $$ipfs_deploy | ||
config: | ||
postUrl: https://wraps.wrapscan.io/r/polywrap/[email protected].0 | ||
postUrl: https://wraps.wrapscan.io/r/polywrap/[email protected].1 | ||
headers: | ||
- name: Authorization | ||
value: $POLYWRAP_WRAPSCAN_AUTH_HEADER_PROD |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
format: 0.1.0 | ||
description: Scrape text and links from website urls | ||
website: https://polywrap.io | ||
repository: https://github.com/polywrap/web-scraper | ||
readme: ./README.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,5 +2,5 @@ | |
|
||
type Module { | ||
get_text(url: String!): String! | ||
get_links(uri: String!): String! | ||
get_links(url: String!): String! | ||
} |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,9 @@ | ||
format: 0.3.0 | ||
format: 0.5.0 | ||
project: | ||
name: web-scraper | ||
type: wasm/rust | ||
source: | ||
module: ./Cargo.toml | ||
schema: ./polywrap.graphql | ||
extensions: | ||
build: ./polywrap.build.yaml | ||
docs: ./polywrap.docs.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,45 +1,49 @@ | ||
import { PolywrapClient } from "@polywrap/client-js"; | ||
import * as App from "../types/wrap"; | ||
import path from "path"; | ||
|
||
jest.setTimeout(60000); | ||
import { WebScraper } from "../types/wrap"; | ||
|
||
describe("Web Scraper Wrapper End to End Tests", () => { | ||
jest.setTimeout(60000); | ||
|
||
const client: PolywrapClient = new PolywrapClient(); | ||
let wrapperUri: string; | ||
describe("WebScraper", () => { | ||
|
||
beforeAll(() => { | ||
const dirname: string = path.resolve(__dirname); | ||
const wrapperPath: string = path.join(dirname, "..", "..", ".."); | ||
wrapperUri = `fs/${wrapperPath}/build`; | ||
}) | ||
const client = new PolywrapClient(); | ||
const localWrap = `file://${__dirname}/../../../build` | ||
|
||
it("calls get_links", async () => { | ||
const uri = "https://polywrap.io"; | ||
const webScraper = new WebScraper(client, undefined, localWrap); | ||
|
||
const result = await client.invoke<App.Module_GetLinks>({ | ||
uri: wrapperUri, | ||
method: "get_links", | ||
args: { uri: uri } | ||
it("get_links", async () => { | ||
const result = await webScraper.get_links({ | ||
url: "https://webscraper.io/test-sites/e-commerce/allinone" | ||
}); | ||
|
||
expect(result.ok).toBeTruthy(); | ||
if (!result.ok) return; | ||
// Add additional assertions as needed | ||
expect(result.value).toContain(`/ | ||
#page-top | ||
/ | ||
/cloud-scraper | ||
/pricing | ||
#section3 | ||
/documentation | ||
/tutorials | ||
/how-to-videos | ||
/test-sites | ||
https://forum.webscraper.io/ | ||
https://chrome.google.com/webstore/detail/web-scraper/jnhgnonknehpejjnehehllkliplmbmhn?hl=en | ||
https://cloud.webscraper.io/ | ||
/test-sites/e-commerce/allinone | ||
/test-sites/e-commerce/allinone/phones | ||
/test-sites/e-commerce/allinone/computers` | ||
); | ||
}); | ||
|
||
it("calls get_text", async () => { | ||
const url = "https://polywrap.io"; | ||
|
||
const result = await client.invoke<App.Module_GetText>({ | ||
uri: wrapperUri, | ||
method: "get_text", | ||
args: { url: url } | ||
it("get_text", async () => { | ||
const result = await webScraper.get_text({ | ||
url: "\nhttps://webscraper.io/test-sites/e-commerce/allinone\n" | ||
}); | ||
|
||
expect(result.ok).toBeTruthy(); | ||
if (!result.ok) return; | ||
// Add additional assertions as needed | ||
expect(result.value).toContain( | ||
`Web Scraper\nCloud Scraper\n` | ||
); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,9 @@ | ||
format: 0.3.0 | ||
format: 0.4.0 | ||
project: | ||
name: sample-typescript-type-generation | ||
type: app/typescript | ||
source: | ||
schema: ./schema.graphql | ||
import_abis: | ||
- uri: "wrap://ens/sample.eth" | ||
- uri: "wrap://wrapscan.io/polywrap/web-scraper" | ||
abi: "../../../build/wrap.info" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
#import * into Template from "wrap://ens/sample.eth" | ||
#import * into WebScraper from "wrap://wrapscan.io/polywrap/web-scraper" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.