diff --git a/.gitignore b/.gitignore index beea512..709fff9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ build/ package-lock.json +*-bundle.js diff --git a/Dockerfile b/Dockerfile index 77b826b..12db7de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY server server COPY .logo-ascii .logo-ascii # Build frontend and install backend dependencies -RUN npm deps && npm run build && rm -rf frontend +RUN npm run deps && npm run build && rm -rf frontend EXPOSE 3000 diff --git a/Dockerfile-local b/Dockerfile-local index c35b71c..255e91c 100644 --- a/Dockerfile-local +++ b/Dockerfile-local @@ -1,26 +1,25 @@ FROM ubuntu:20.04 # Install Node.js -RUN apt-get update && apt-get install -y --reinstall ca-certificates curl build-essential -RUN curl --silent --location https://deb.nodesource.com/setup_12.x | bash - +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update RUN apt-get install -y nodejs -RUN npm install -g npm@6.10.0 +RUN node --version -# Copy bundled frontend +# Copy bundled frontend and backend dependencies COPY build build +COPY node_modules node_modules # Copy files for the backend COPY package.json package.json COPY server server COPY .logo-ascii .logo-ascii -# Install backend dependencies -RUN npm install EXPOSE 3000 # default files and folders (usefull when no volume can be mounted with this image) RUN mkdir -p /data - +COPY data-test /data/data-test # ENTRYPOINT ["node", "server/server.js"] RUN echo 'cat .logo-ascii && node server/server.js "$@"' > entrypoint.sh diff --git a/README.md b/README.md index fb4c670..653cec9 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ pixano ./data-test --port 3001 #### Install global dependencies -- NodeJS (>=12) +- NodeJS (10, 12 or 14) To install on ubuntu: ```bash @@ -78,6 +78,8 @@ nodejs --version npm install -g npm@6.10.0 ``` You can read this nice [introduction](https://codeburst.io/the-only-nodejs-introduction-youll-ever-need-d969a47ef219) to NodeJS in case you're curious on how it works. + +> ATTENTION: node version 16 is not compatible for now #### Install application dependencies @@ -90,10 +92,16 @@ If you want to use custom `pixano-element` modules from local path instead of th ```bash # Install application dependencies and local pixano-elements -npm run installLocalElements --path=../../pixano-elements +npm run installLocalElements --path=$PIXANO_ELEMENTS_PATH ``` *NB: Make sure you have the git repository of pixano-elements next to the pixano-app folder and that you have followed the pixano-elements build instructions before running the above commands.* +If this command breaks your local pixano-elements demo, this command will repear it: +```bash +cd $PIXANO_ELEMENTS_PATH +npm run bootstrap +``` + #### Build the application ```bash @@ -187,10 +195,13 @@ The `task1.json` file contains global task settings (task type, task categories, ### Build docker from sources -To create a docker image of the application, build the application (step 1.b) and then run: +To create a docker image of the application, you can use the standard docker command: ```bash # You can change `pixano` by your choosen image name sudo docker build -t pixano/pixano-app:my-tag . +``` +If you used a local pixano-element, build the application (step 1.b) and then run: +```bash # You can use the local Dockerfile if the build folder already exists sudo docker build -t pixano/pixano-app:my-tag -f Dockerfile-local . ``` diff --git a/documentation/rest-api.md b/documentation/rest-api.md index 96a9e65..37c2b98 100644 --- a/documentation/rest-api.md +++ b/documentation/rest-api.md @@ -57,7 +57,7 @@ This document lists the API of the Pixano server that enable the creation of new type Objective = 'to_annotate' | 'to_validate' | 'to_correct'; // possible status a data item can have -type LabellingStatus = Objective | 'done'; +type LabellingStatus = Objective | 'done' | 'discard'; // possible types a data item can have type DataType = 'image' | 'pcl' | 'pcl_image' | 'sequence_pcl' | 'sequence_image' | 'sequence_pcl_image'; diff --git a/frontend/package.json b/frontend/package.json index d823233..780816a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pixano-app-frontend", - "version": "0.5.0", + "version": "0.6.0", "description": "This is a Pixano app.", "scripts": { "copyindex": "shx cp src/index.html ../build", @@ -49,10 +49,10 @@ "@material/mwc-tab-bar": "0.19.1", "@material/mwc-textarea": "0.19.1", "@material/mwc-textfield": "0.19.1", - "@pixano/ai": "0.6.1", - "@pixano/core": "0.6.1", - "@pixano/graphics-2d": "0.6.1", - "@pixano/graphics-3d": "0.6.1", + "@pixano/ai": "0.7.0", + "@pixano/core": "0.7.0", + "@pixano/graphics-2d": "0.7.0", + "@pixano/graphics-3d": "0.7.0", "@trystan2k/fleshy-jsoneditor": "3.0.0", "@webcomponents/webcomponentsjs": "^2.4.0", "babel-loader": "^8.2.3", diff --git a/frontend/src/actions/requests.js b/frontend/src/actions/requests.js index 74dedef..2760526 100644 --- a/frontend/src/actions/requests.js +++ b/frontend/src/actions/requests.js @@ -29,7 +29,7 @@ const _requestHelper = (method, url = "/api/v1/", body = undefined, dispatch = n dispatch(updateWaiting(true)); } return new Promise((resolve, reject) => { - return fetch(url, messageContent).then((response) => { + return fetch(encodeURI(url), messageContent).then((response) => { if (dispatch) { dispatch(updateWaiting(false)); } diff --git a/frontend/src/app.js b/frontend/src/app.js index 6764c57..2d8ebed 100644 --- a/frontend/src/app.js +++ b/frontend/src/app.js @@ -68,13 +68,13 @@ class MyApp extends connect(store)(LitElement) { goHome() { const user = getState('user'); const page = user.currentUser.role === 'admin' ? '/#dashboard-admin': '/#dashboard-user'; - window.history.pushState({}, '', page); + window.history.pushState({}, '', encodeURI(page)); store.dispatch(navigate(page)); } goLogin() { const page = '/#login'; - window.history.pushState({}, '', page); + window.history.pushState({}, '', encodeURI(page)); store.dispatch(navigate(page)); } diff --git a/frontend/src/helpers/attribute-picker.js b/frontend/src/helpers/attribute-picker.js deleted file mode 100644 index eadeaf2..0000000 --- a/frontend/src/helpers/attribute-picker.js +++ /dev/null @@ -1,407 +0,0 @@ -/** - * Utility class to pick labels in a panel - * @copyright CEA-LIST/DIASI/SIALV/LVA (2019) - * @author CEA-LIST/DIASI/SIALV/LVA - * @license CECILL-C -*/ - -import { LitElement, html, css } from 'lit-element'; -import '@material/mwc-dialog'; -import '@material/mwc-checkbox'; -import '@material/mwc-formfield'; -import '@material/mwc-select'; -import '@material/mwc-list/mwc-list-item'; -import { getValue } from '../helpers/utils'; - - -// TODO: move to pixano-elements -export class AttributePicker extends LitElement { - - static get styles() { - return [ - css` - :host { - -webkit-touch-callout: none; /* iOS Safari */ - -webkit-user-select: none; /* Safari */ - -khtml-user-select: none; /* Konqueror HTML */ - -moz-user-select: none; /* Old versions of Firefox */ - -ms-user-select: none; /* Internet Explorer/Edge */ - user-select: none; /* Non-prefixed version, currently - supported by Chrome, Opera and Firefox */ - } - h3 { - font-size: 14px; - margin-left: 10px; - } - .category { - height: 40px; - display: flex; - align-items: center; - padding-left: 10px; - } - .category:hover { - background-color: #ececec; - cursor: pointer; - } - .selected { - background-color: rgb(230, 230, 230); - color: var(--secondary-color); - } - span.step { - background: red; - border-radius: 0.8em; - -moz-border-radius: 0.8em; - -webkit-border-radius: 0.8em; - color: #ffffff; - display: inline-block; - line-height: 1.6em; - margin-right: 15px; - text-align: center; - width: 1.6em; - margin-left: 10px; - } - .category > p { - margin: 0; - padding-left: 10px; - } - .shortcut { - position: absolute; - right: 0px; - z-index: 1; - } - #shortcut-table { - font-family: "Trebuchet MS", Arial, Helvetica, sans-serif; - border-collapse: collapse; - width: 100%; - } - - #shortcut-table td, #shortcut-table th { - border: 1px solid #ddd; - padding: 8px; - } - - #shortcut-table tr:nth-child(even){background-color: #f2f2f2;} - - #shortcut-table tr:hover {background-color: #ddd;} - - #shortcut-table th { - padding-top: 12px; - padding-bottom: 12px; - text-align: left; - background-color: #4CAF50; - color: white; - } - mwc-select { - width: 100%; - } - mwc-formfield { - margin: auto; - width: 70%; - display: flex; - } - ` - ] - } - - static get properties () { - /** - * showDetail: Boolean, rendering mode for the selected category (showing all attributes or only the category) - * shortcuts : Array of strings, contains the list of all applicable keyboard shortcuts - * schema: shema for this annotation (i.e. category and attributes available for each category in this annotation) - * value: {category, options }, contains the value of the current category and its options (i.e. attributes available for this category) - * numDone: Number, only used for keypoints-box - * numTotal: Number, only used for keypoints-box - */ - return { - showDetail: { type: Boolean }, - shortcuts: { type: Array }, - schema: { type: Object }, - value: { type: Object }, - numDone: { type: Number }, - numTotal: { type: Number } - } - } - - get selectedCategory() { - return this.schema.category.find((c) => c.name === this.value.category); - } - - getDefaultAttributesForCategory(schema, categoryName) { - let category = schema.category.find((c) => c.name === categoryName); - if (!category && schema.category.length) { - category = schema.category[0]; - } - if (category && category.properties) { - const d = {}; - category.properties.forEach((p) => { - d[p.name] = p.default; - }); - return d; - } - return {} - } - - onKeyDown(event) { - if (event.ctrlKey) { - event.preventDefault(); - } - } - - onKeyUp(event) { - const isNumber = event.code.replace('Digit', '').replace('Numpad', '') - if (Number(isNumber) >= 0 && Number(isNumber) <= 9 && event.ctrlKey) { - event.preventDefault(); - this.mem += isNumber; - - } - if (event.key === 'Control' && this.mem !== '') { - event.preventDefault(); - const c = this.schema.category[Number(this.mem)]; - if (c) { - this.setCategory(c.name); - } - this.mem = ''; - } - } - - constructor() { - super(); - this.shortcuts = [ - ['ALT', 'Switch create/update mode'], - ['CTRL + [0-9]', 'Select category by index'], - ['TAB', 'Navigate through objects'], - ['SHIFT + Tab', 'Navigate through objects (inverse)'], - ['SHIFT + Click', 'Multiple selection'], - ['CTRL + z', 'Undo'], - ['CTRL + SHIFT + z', 'Redo'], - ['CTRL + s', 'Save'] - ]; - this.showDetail = false; - this.mem = ''; - this.schema = {}; - this.schema.category = []; - const options = {}; - this.value = {category: '', options }; - this.onKeyDown = this.onKeyDown.bind(this); - this.onKeyUp = this.onKeyUp.bind(this); - } - - connectedCallback() { - super.connectedCallback(); - window.addEventListener('keydown', this.onKeyDown); - window.addEventListener('keyup', this.onKeyUp); - } - - disconnectedCallback() { - super.disconnectedCallback(); - window.removeEventListener('keydown', this.onKeyDown); - window.removeEventListener('keyup', this.onKeyUp); - } - - openShortcuts() { - const d = this.shadowRoot.querySelector('mwc-dialog'); - d.open = true; - } - - _getList() { - try { - return this.schema.category.map((c)=> c.name); - } catch { - return []; - } - } - - _colorFor(categoryName) { - const category = this.schema.category.find((c) => c.name === categoryName); - return category ? category.color || 'rgb(0,0,0)' : 'rgb(0,0,0)'; - } - - get defaultValue() { - const options = this.getDefaultAttributesForCategory(this.schema, this.value.category); - return {category: this.value.category, options}; - } - - setCategory(newCategory) { - const options = this.getDefaultAttributesForCategory(this.schema, newCategory); - this.value = {category: newCategory, options }; - this._notifyUpdate(); - } - - /** - * Triggered when exterior/non-user-triggered - * edition of edition label schema - * @param {*} entity - */ - setAttributes(entity) { - if (entity) { - entity.options = entity.options || {}; - const options = this.getDefaultAttributesForCategory(this.schema, entity.category); - Object.keys(options).forEach((key) => { - if (entity.options.hasOwnProperty(key)) { - options[key] = JSON.parse(JSON.stringify(entity.options[key])); - } else { - options[key] = ""; - } - }); - // update property choices - this.value = {category: entity.category, options}; - - // update property values - const childDivs = this.shadowRoot.getElementById('updateEditor').getElementsByTagName('mwc-select'); - const childContent = this.schema.category.find((c) => c.name === this.value.category); - const propList = childContent ? childContent.properties : []; - [ ...childDivs].forEach((c) => { - const subprop = propList.find((p) => p.name == c.label); - const enumList = subprop ? subprop.enum : []; - // cast to number - const val = isNaN(options[c.label]) ? options[c.label] : parseInt(options[c.label]); - c.select(enumList.findIndex((e) => e === val)) - }); - } - } - - setAttributesIdx(idx) { - if (idx != undefined) { - this.value = {category: this.schema.category.find((c) => c.idx === idx).name}; - } else { - const options = this.getDefaultAttributesForCategory(this.schema, this.schema.default); - this.value = {category: this.schema.default, options }; - } - } - - reloadSchema(schema) { - this.schema = schema; - const options = this.getDefaultAttributesForCategory(schema, schema.default); - this.value = {category: schema.default, options }; - } - - _notifyUpdate() { - this.dispatchEvent(new Event('update')); - } - - get shortcutsDialog() { - return html` - -

Shortcut list

-
- - - - - - ${ - this.shortcuts.map(([k,v]) => { - return html` - - `; - }) - } -
ShortcutDescription
${k}${v}
-
- OK - - ` - } - - firstUpdated() { - this.reloadSchema(this.schema); - } - - htmlProp(prop) { - if (prop.type === 'dropdown') { - // list of attribute choices - return html` - { - const idx = e.detail.index; - if (this.value.options[prop.name] != prop.enum[idx]) { - this.value.options[prop.name] = prop.enum[idx]; - this._notifyUpdate(); - } - }}> - ${prop.enum.map((sub) => { - return html`${sub}` - })} - - ` - } else if (prop.type === 'checkbox') { - const checked = JSON.parse(JSON.parse(JSON.stringify(this.value.options[prop.name]).toLowerCase()));// if the initial value was a string like "false" or "False", we want it to be interpreted as a boolean - return html` - - { - const path = evt.composedPath(); - const input = path[0]; - if (checked != input.checked) { - this.value.options[prop.name] = !checked; - this.value = {...this.value}; - this._notifyUpdate(); - } - } - }> - - ` - } else if (prop.type === 'textfield') { - const textval = this.value.options[prop.name]; - return html` - { - this.value.options[prop.name] = getValue(evt); - this.value = {...this.value}; - this._notifyUpdate(); - } - }> - ` - } - return html``; - } - - get renderDetail() { - return html` -
-

- ${ - this.schema.category.map((category, idx) => { - return html` -
this.setCategory(category.name)}> - ${idx}

${category.name}

-
- ${category.properties && category.name === this.value.category ? html` - ${category.properties.map((prop) => this.htmlProp(prop))} - `: html``} - ` - }) - } -
` - } - - get renderSimple() { - return html` -
-

- ${ - this.schema.category.map((category, idx) => { - return html` -
this.setCategory(category.name)}> - ${idx}

${category.name}

-
` - }) - } -
- `; - } - - /** - * Render the element template. - */ - render(){ - return html` - ${this.shortcutsDialog} - - ${this.renderDetail} - ${this.renderSimple} - `; - } -} - -customElements.define('attribute-picker', AttributePicker); diff --git a/frontend/src/helpers/utils.js b/frontend/src/helpers/utils.js index 2886331..0a25e2a 100644 --- a/frontend/src/helpers/utils.js +++ b/frontend/src/helpers/utils.js @@ -4,169 +4,6 @@ * @license CECILL-C */ -export function commonJson(entities) { - if (entities.length == 0) { - return; - } else if (entities.length == 1) { - return entities[0]; - } - function getKeys(object) { - function iter(o, p) { - if (Array.isArray(o)) { - result.push(p.join('.')); - return; - } - if (o && typeof o === 'object') { - var keys = Object.keys(o); - if (keys.length) { - keys.forEach((k) => { iter(o[k], p.concat(k)); }); - } - return; - } - result.push(p.join('.')); - } - var result = []; - iter(object, []); - return result; - } - const commonEntity = JSON.parse(JSON.stringify(entities)).shift(); - const keys = getKeys(commonEntity); - keys.forEach((key) => { - const commonAttr = key.split('.').reduce((o, p) => (o && o.hasOwnProperty(p)) ? o[p] : null, commonEntity); - for (const e of entities) { - const entityAttr = key.split('.').reduce((o, p) => (o && o.hasOwnProperty(p)) ? o[p] : null, e); - if (!entityAttr || JSON.stringify(entityAttr) != JSON.stringify(commonAttr)) { - // remove from commonEntity - deleteObjProp(commonEntity, key); - break; - } - } - }); - return commonEntity; -} - -function deleteObjProp(obj, path) { - if (!obj || !path) { - return; - } - if (typeof path === 'string') { - path = path.split('.'); - } - for (var i = 0; i < path.length - 1; i++) { - obj = obj[path[i]]; - if (typeof obj === 'undefined') { - return; - } - } - delete obj[path.pop()]; -}; - -export function colorToRGBA(color) { - // Returns the color as an array of [r, g, b, a] -- all range from 0 - 255 - // color must be a valid canvas fillStyle. This will cover most anything - // you'd want to use. - // Examples: - // colorToRGBA('red') # [255, 0, 0, 255] - // colorToRGBA('#f00') # [255, 0, 0, 255] - var cvs, ctx; - cvs = document.createElement('canvas'); - cvs.height = 1; - cvs.width = 1; - ctx = cvs.getContext('2d'); - ctx.fillStyle = color; - ctx.fillRect(0, 0, 1, 1); - return ctx.getImageData(0, 0, 1, 1).data; -} - -function byteToHex(num) { - // Turns a number (0-255) into a 2-character hex number (00-ff) - return ('0'+num.toString(16)).slice(-2); -} - -export function colorToHex(color) { - // Convert any CSS color to a hex representation - // Examples: - // colorToHex('red') # '#ff0000' - // colorToHex('rgb(255, 0, 0)') # '#ff0000' - var rgba, hex; - rgba = colorToRGBA(color); - hex = [0,1,2].map( - function(idx) { return byteToHex(rgba[idx]); } - ).join(''); - return "#"+hex; -} - -export const isEqual = (value, other) => { - - // Get the value type - var type = Object.prototype.toString.call(value); - - // If the two objects are not the same type, return false - if (type !== Object.prototype.toString.call(other)) return false; - - // If items are not an object or array, return false - if (['[object Array]', '[object Object]'].indexOf(type) < 0) return false; - - // Compare the length of the length of the two items - var valueLen = type === '[object Array]' ? value.length : Object.keys(value).length; - var otherLen = type === '[object Array]' ? other.length : Object.keys(other).length; - if (valueLen !== otherLen) return false; - - // Compare two items - var compare = function (item1, item2) { - - // Get the object type - var itemType = Object.prototype.toString.call(item1); - - // If an object or array, compare recursively - if (['[object Array]', '[object Object]'].indexOf(itemType) >= 0) { - if (!isEqual(item1, item2)) return false; - } - - // Otherwise, do a simple comparison - else { - - // If the two items are not the same type, return false - if (itemType !== Object.prototype.toString.call(item2)) return false; - - // Else if it's a function, convert to a string and compare - // Otherwise, just compare - if (itemType === '[object Function]') { - if (item1.toString() !== item2.toString()) return false; - } else { - if (item1 !== item2) return false; - } - - } - }; - - // Compare properties - if (type === '[object Array]') { - for (var i = 0; i < valueLen; i++) { - if (compare(value[i], other[i]) === false) return false; - } - } else { - for (var key in value) { - if (value.hasOwnProperty(key)) { - if (compare(value[key], other[key]) === false) return false; - } - } - } - - // If nothing failed, return true - return true; - -}; - -export function getProperty(obj, path) { - return path.split('.').reduce((p,c)=>p&&p[c]||null, obj); -} - -export function setProperty(obj, path, value) { - path.split('.') - .reduce((o,p,i) => o[p] = path.split('.').length === ++i ? value : o[p] || {}, obj) -} - /** * Retrieve input text value from change event * @param {Event} e diff --git a/frontend/src/plugins/classification.js b/frontend/src/plugins/classification.js index 22af5b1..a2d7dec 100644 --- a/frontend/src/plugins/classification.js +++ b/frontend/src/plugins/classification.js @@ -6,8 +6,8 @@ import { html } from 'lit-element'; import '@pixano/graphics-2d/lib/pxn-classification'; +import '@pixano/core/lib/attribute-picker'; import { TemplatePlugin } from '../templates/template-plugin'; -import '../helpers/attribute-picker'; import { store } from '../store'; import { setAnnotations } from '../actions/annotations'; @@ -24,8 +24,11 @@ export class PluginClassification extends TemplatePlugin { refresh() { if (!this.element) return; - if (this.annotations.length===0) this.attributePicker.setAttributes(this.attributePicker.defaultValue);// initialize to default - else { + this.attributePicker.showDetail = true;// exception for classification: always show details + if (this.annotations.length===0) {// initialize to default + this.attributePicker.setAttributes(this.attributePicker.defaultValue); + store.dispatch(setAnnotations({annotations: [this.attributePicker.value]}));//Save current state to redux database (to keep history) + } else { this.element.annotations = JSON.parse(JSON.stringify(this.annotations)); this.attributePicker.setAttributes(this.element.annotations[0]); } @@ -44,7 +47,7 @@ export class PluginClassification extends TemplatePlugin { * Implement property panel content, details always visible */ get propertyPanel() { - return html`` + return html`` } get editor() { diff --git a/frontend/src/plugins/index.js b/frontend/src/plugins/index.js index 671b534..7ed6cf3 100644 --- a/frontend/src/plugins/index.js +++ b/frontend/src/plugins/index.js @@ -7,9 +7,6 @@ * @license CECILL-C */ -import '../helpers/attribute-picker'; - - /** * List of all plugin names */ diff --git a/frontend/src/plugins/keypoints-box.js b/frontend/src/plugins/keypoints-box.js index 25aca4e..211b59e 100644 --- a/frontend/src/plugins/keypoints-box.js +++ b/frontend/src/plugins/keypoints-box.js @@ -5,7 +5,7 @@ */ import { html } from 'lit-element'; -import { settings } from '@pixano/graphics-2d/lib/pxn-graph'; +import { settings } from '@pixano/graphics-2d/lib/pxn-keypoints'; import '@material/mwc-icon-button'; import { colorNames, shuffle } from '@pixano/core/lib/utils' import { TemplatePluginInstance } from '../templates/template-plugin-instance'; @@ -160,13 +160,13 @@ export class PluginKeypointsBox extends TemplatePluginInstance { } get editor() { - return html``; + @selection=${this.onSelection}>`; } } customElements.define('plugin-keypoints-box', PluginKeypointsBox); diff --git a/frontend/src/plugins/keypoints.js b/frontend/src/plugins/keypoints.js index ba233f3..f029221 100644 --- a/frontend/src/plugins/keypoints.js +++ b/frontend/src/plugins/keypoints.js @@ -78,13 +78,13 @@ export class PluginKeypoints extends TemplatePluginInstance { } get editor() { - return html``; + @mode=${this.onModeChange}>`; } } customElements.define('plugin-keypoints', PluginKeypoints); diff --git a/frontend/src/plugins/segmentation.js b/frontend/src/plugins/segmentation.js index 3aec0d1..21d475a 100644 --- a/frontend/src/plugins/segmentation.js +++ b/frontend/src/plugins/segmentation.js @@ -6,16 +6,15 @@ import { html } from 'lit-element'; import '@pixano/graphics-2d/lib/pxn-segmentation'; +import '@pixano/core/lib/attribute-picker'; import '@material/mwc-icon-button'; import '@material/mwc-icon-button-toggle'; import '@material/mwc-icon'; -import { colorToRGBA } from '@pixano/core/lib/utils'; +import { colorToRGBA, commonJson } from '@pixano/core/lib/utils'; import { store, getState } from '../store'; -import '../helpers/attribute-picker'; import { subtract, union } from '../my-icons'; import { setAnnotations } from '../actions/annotations'; import { TemplatePluginInstance } from '../templates/template-plugin-instance'; -import { commonJson } from '../helpers/utils'; const EditionMode = { ADD_TO_INSTANCE: 'add_to_instance', diff --git a/frontend/src/templates/template-page.js b/frontend/src/templates/template-page.js index 5356f92..984d4cf 100644 --- a/frontend/src/templates/template-page.js +++ b/frontend/src/templates/template-page.js @@ -87,7 +87,7 @@ export default class TemplatePage extends LitElement { * @param {string} page e.g. /#dashboard-admin or / */ gotoPage(page) { - window.history.pushState({}, '', page); + window.history.pushState({}, '', encodeURI(page)); store.dispatch(navigate(page)); } @@ -106,7 +106,7 @@ export default class TemplatePage extends LitElement { } else { return; } - window.history.pushState({}, '', page); + window.history.pushState({}, '', encodeURI(page)); store.dispatch(navigate(page)); } diff --git a/frontend/src/templates/template-plugin-instance.js b/frontend/src/templates/template-plugin-instance.js index 621e7d8..dfb88be 100644 --- a/frontend/src/templates/template-plugin-instance.js +++ b/frontend/src/templates/template-plugin-instance.js @@ -9,10 +9,10 @@ import { html } from 'lit-element'; import '@material/mwc-icon-button'; import '@material/mwc-icon-button-toggle'; import '@material/mwc-icon'; -import { commonJson } from '../helpers/utils'; +import { commonJson } from '@pixano/core/lib/utils'; import { store } from '../store'; import { setAnnotations } from '../actions/annotations'; -import '../helpers/attribute-picker'; +import '@pixano/core/lib/attribute-picker'; import { TemplatePlugin } from './template-plugin'; export class TemplatePluginInstance extends TemplatePlugin { @@ -127,7 +127,7 @@ export class TemplatePluginInstance extends TemplatePlugin { */ get propertyPanel() { return html` - ` } diff --git a/frontend/src/views/app-dashboard-admin.js b/frontend/src/views/app-dashboard-admin.js index f9fe7b7..a92254e 100644 --- a/frontend/src/views/app-dashboard-admin.js +++ b/frontend/src/views/app-dashboard-admin.js @@ -58,8 +58,8 @@ class AppDashboardAdmin extends TemplatePage { ['to_annotate', ['to annotate', 'create', 'blue']], ['to_validate', ['to validate', 'youtube_searched_for', 'orange']], ['to_correct', ['to correct', 'thumb_down', 'red']], + ['discard', ['do NOT annotate', 'highlight_off', 'red']], ['done', ['done', 'done', 'green']]]); - this.assignedMap = new Map([['', ''], ['true', 'in progress'], ['false', 'idle']]); @@ -467,7 +467,7 @@ class AppDashboardAdmin extends TemplatePage { /** * Display table row - * Status | Data Id | Annotator | Validator | State | Time | Thumbnail | Launch + * Status | Data Id | Annotator | Validator | State | Time | Thumbnail */ listitem(item) { const v = this.statusMap.get(item.status); @@ -483,8 +483,8 @@ class AppDashboardAdmin extends TemplatePage {

${item.validator}

${this.assignedMap.get(item.in_progress.toString())}

${format(item.cumulated_time)}

-

-

this.onExplore(evt, item.data_id)}>

+

this.onExplore(evt, item.data_id)}>

+

  • diff --git a/frontend/src/views/app-explore.js b/frontend/src/views/app-explore.js index 38792c5..5190235 100644 --- a/frontend/src/views/app-explore.js +++ b/frontend/src/views/app-explore.js @@ -39,7 +39,7 @@ export class AppExplore extends TemplatePage { onActivate() { const paths = window.location.hash.split('/'); - const taskName = paths[1]; + const taskName = decodeURI(paths[1]); const dataId = paths[2]; store.dispatch(updateTaskName(taskName)); const task = getState('application').tasks.find((t) => t.name === taskName); @@ -102,7 +102,8 @@ export class AppExplore extends TemplatePage { const appState = getState('application'); const nextDataId = appState.dataId; if (nextDataId !== currentDataId) { - window.history.pushState({}, '', `/#explore/${appState.taskName}/${nextDataId}`); + const page = '/#explore/'+appState.taskName+'/'+nextDataId; + window.history.pushState({}, '', encodeURI(page)); this.dataPath = this.path; } }); diff --git a/frontend/src/views/app-label.js b/frontend/src/views/app-label.js index d3d4a45..08979db 100644 --- a/frontend/src/views/app-label.js +++ b/frontend/src/views/app-label.js @@ -52,7 +52,7 @@ class AppLabel extends AppExplore { onActivate() { const paths = window.location.hash.split('/'); - const taskName = paths[1]; + const taskName = decodeURI(paths[1]); this.jobObjective = paths[2] || this.jobDefaultObjective; store.dispatch(updateTaskName(taskName)); const task = getState('application').tasks.find((t) => t.name === taskName); diff --git a/frontend/src/views/app-login.js b/frontend/src/views/app-login.js index 70e5bb6..a02c570 100644 --- a/frontend/src/views/app-login.js +++ b/frontend/src/views/app-login.js @@ -37,7 +37,7 @@ class AppLogin extends LitElement { goHome() { const user = getState('user'); const page = user.currentUser.role === 'admin' ? '/#dashboard-admin': '/#dashboard-user'; - window.history.pushState({}, '', page); + window.history.pushState({}, '', encodeURI(page)); store.dispatch(navigate(page)); } diff --git a/frontend/src/views/app-project-manager.js b/frontend/src/views/app-project-manager.js index 0150694..ef89b03 100644 --- a/frontend/src/views/app-project-manager.js +++ b/frontend/src/views/app-project-manager.js @@ -204,12 +204,12 @@ class AppProjectManager extends connect(store)(TemplatePage) { */ SaveOrCreateTask() { let task = { ...this.tasks[this.taskIdx] }; - const reg = new RegExp('[^=a-zA-Z0-9-_]+',); - const isWrongContained = reg.test(task.name); - if (!task.name || isWrongContained) { - this.errorPopup("Enter a correct task name"); - return; - } + // const reg = new RegExp('[^=a-zA-Z0-9-_]+',); + // const isWrongContained = reg.test(task.name); + // if (!task.name || isWrongContained) { + // this.errorPopup("Enter a correct task name"); + // return; + // } task.spec.label_schema = this.taskSettings.json; task.spec.settings = this.pluginSettings.json; const fn = this.creatingTask ? postTask : putTask; diff --git a/package.json b/package.json index 06278f5..93fa3b6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pixano-app", - "version": "0.5.0", + "version": "0.6.0", "description": "This is a Pixano app.", "keywords": [], "license": "CECILL-C", @@ -10,6 +10,7 @@ }, "scripts": { "deps": "rm -f package-lock.json && npm i && cd frontend && rm -f package-lock.json && npm i", + "clearall": "rm -rf node_modules/ package-lock.json frontend/node_modules/ frontend/package-lock.json", "build": "cd frontend && npm run build", "installLocalElements": "cd frontend && npm run installLocalElements" }, diff --git a/server/routes/jobs.js b/server/routes/jobs.js index 29b5cb1..6452ae5 100644 --- a/server/routes/jobs.js +++ b/server/routes/jobs.js @@ -47,12 +47,13 @@ async function get_next_job(req, res) { const assignedJob = user.curr_assigned_jobs[oKey]; if (assignedJob) { let job, result; + var error=false; try { job = await db.get(dbkeys.keyForJob(taskName, assignedJob)); result = await db.get(dbkeys.keyForResult(taskName, job.data_id)); } - catch (err) { console.log('\tjob no longer exist', err);} - if (isJobValid(job, result) && await isJobAvailableForUser(job, user)) { + catch (err) { error=true; console.log('\tjob no longer exist', err);} + if (!error && isJobValid(job, result) && await isJobAvailableForUser(job, user)) { await _assignJob(job, user); return res.send({...job, annotator: result.annotator, validator: result.validator}); } @@ -68,12 +69,13 @@ async function get_next_job(req, res) { if (queuedJobs.length) { for (const q of queuedJobs) { let job, result; + var error=false; try { job = await db.get(dbkeys.keyForJob(taskName, q)); result = await db.get(dbkeys.keyForResult(taskName, job.data_id)); } - catch (err) { console.log('\tjob no longer exist', err); } - if (isJobValid(job, result) && await isJobAvailableForUser(job, user)) { + catch (err) { error=true; console.log('\tjob no longer exist', err); } + if (!error && isJobValid(job, result) && await isJobAvailableForUser(job, user)) { await _assignJob(job, user); return res.send({...job, annotator: result.annotator, validator: result.validator}); } @@ -90,9 +92,10 @@ async function get_next_job(req, res) { for await(const result of streamA) { if (objectiveList.includes(result.status) && !result.in_progress) { let job; + var error=false; try { job = await db.get(dbkeys.keyForJob(taskName, result.current_job_id)); } - catch (err) { console.log('\tjob no longer exist', err); } - if (isJobValid(job, result) && await isJobAvailableForUser(job, user)) { + catch (err) { error=true; console.log('\tjob no longer exist', err); } + if (!error && isJobValid(job, result) && await isJobAvailableForUser(job, user)) { await _assignJob(job, user); return res.send({...job, annotator: result.annotator, validator: result.validator}); } @@ -103,9 +106,10 @@ async function get_next_job(req, res) { for await(const result of streamB) { if (objectiveList.includes(result.status) && !result.in_progress) { let job; + var error=false; try { job = await db.get(dbkeys.keyForJob(taskName, result.current_job_id)); } - catch (err) { console.log('\tjob no longer exist', err); } - if (isJobValid(job, result) && await isJobAvailableForUser(job, user, true)) { + catch (err) { error=true; console.log('\tjob no longer exist', err); } + if (!error && isJobValid(job, result) && await isJobAvailableForUser(job, user, true)) { await _assignJob(job, user); return res.send({...job, annotator: result.annotator, validator: result.validator}); } @@ -205,17 +209,17 @@ async function put_job(req, res) { // Update result resultData.in_progress = false; - resultData.status = ['to_annotate', 'to_validate', 'to_correct', 'done'].includes(newObjective) ? newObjective : 'to_annotate'; + resultData.status = ['to_annotate', 'to_validate', 'to_correct', 'discard', 'done'].includes(newObjective) ? newObjective : 'to_annotate'; resultData.finished_job_ids.push(resultData.current_job_id); - if (newObjective === 'done') { + if (newObjective === 'done' || newObjective === 'discard') { resultData.current_job_id = ''; ops.push({ type: 'put', key: dbkeys.keyForResult(taskName, jobData.data_id), value: resultData}); await db.batch(ops); return res.status(204).json({}); } - // Create new Job if not 'done' + // Create new Job if not 'done' or 'discard' const newJob = createJob(jobData.task_name, jobData.data_id, newObjective); resultData.current_job_id = newJob.id; diff --git a/server/routes/tasks.js b/server/routes/tasks.js index db6cab4..dc43e5a 100644 --- a/server/routes/tasks.js +++ b/server/routes/tasks.js @@ -15,6 +15,8 @@ const { getAllDataFromDataset, getAllPathsFromDataset, getDataDetails } = require('./datasets'); +const annotation_format_version = "0.9"; + /** * @api {get} /tasks Get list of tasks details * @apiName GetTasks @@ -89,6 +91,12 @@ async function post_tasks(req, res) { */ async function import_tasks(req, res) { checkAdmin(req, async () => { + if (req.body.url) { + return res.status(400).json({ + error: 'url_not_implemented', + message: 'Import tasks from URL is not yet implemented.' + }); + } if (!req.body.path) { return res.status(400).json({ error: 'wrong_path', @@ -124,8 +132,17 @@ async function import_tasks(req, res) { // dataset: { path: string, data_type: string } // } const importedTasks = []; - for await (const jsonf of taskJsonFiles) { + for await (const jsonf of taskJsonFiles) { const taskData = utils.readJSON(path.join(importPath, jsonf)); + let version = taskData.version; + // check annotation format version + if (!version) version = "0.9";//0.9 is the first versioned format + if (parseFloat(version) < parseFloat(annotation_format_version)) { + // TO BE DETERMINED when new version will arrise: solve compatibility issues + } + console.info("Annotation format version:",annotation_format_version); + + const dataset = await getOrcreateDataset({...taskData.dataset, data_type: taskData.spec.data_type}, workspace); const spec = await getOrcreateSpec(taskData.spec); @@ -162,7 +179,7 @@ async function import_tasks(req, res) { // annotations: any[], // data: { type: string, path: string | string[], children: array<{path, timestamp}>} // } - for await (const jsonFile of annJsonFiles) { + for await (const jsonFile of annJsonFiles) { // Create data const fullPath = path.join(importPath, taskFolder, jsonFile); const ann = utils.readJSON(fullPath); @@ -205,6 +222,11 @@ async function import_tasks(req, res) { }; await bm.add({ type: 'put', key: dbkeys.keyForLabels(newTask.name, newLabels.data_id), value: newLabels}); + if (ann.data.status) {//if existing, get status back + resultData = await db.get(dbkeys.keyForResult(newTask.name, newLabels.data_id));//get the status for this data + resultData.status = ann.data.status;//add the status + await bm.add({ type: 'put', key: dbkeys.keyForResult(newTask.name, newLabels.data_id), value: resultData}); + } // Mark result as done // const resultData = await db.get(dbkeys.keyForResult(newTask.name, dataId)); @@ -231,7 +253,7 @@ async function import_tasks(req, res) { * @apiGroup Tasks * @apiPermission admin * - * @apiParam {string} [path] Relative path to tasks folder + * @apiParam {string} [path] Relative path to tasks folder OR [url] destination URL for online export * * @apiSuccessExample Success-Response: * HTTP/1.1 200 OK @@ -240,75 +262,135 @@ async function import_tasks(req, res) { * HTTP/1.1 400 Failed to create export folder */ async function export_tasks(req, res) { - checkAdmin(req, async () => { - if (!req.body.path) { - return res.status(400).json({ - error: 'wrong_path', - message: 'Invalid path.' - }); - } - const exportPath = path.join(workspace, req.body.path); - console.log('##### Exporting to ', exportPath); - if(!utils.isSubFolder(workspace, exportPath)) { - return res.status(400).json({ - error: 'wrong_folder', - message: 'Export folder should be a sub folder of the working space.' - }); - } - // If path does not exist create it - if (!fs.existsSync(exportPath)) { - fs.mkdirSync(exportPath, {recursive: true}); - } - - const streamTask = utils.iterateOnDB(db, dbkeys.keyForTask(), false, true); - for await (const task of streamTask) { - const spec = await db.get(dbkeys.keyForSpec(task.spec_id)); - delete spec.id; - const dataset = await db.get(dbkeys.keyForDataset(task.dataset_id)); - const datasetId = dataset.id; - delete dataset.id; - const taskJson = {name: task.name, spec, dataset}; - - // Write task json - const err = utils.writeJSON(taskJson, `${exportPath}/${task.name}.json`); - if (err) { - return; - } - - // Write annotations for each task in a specific folder - const taskFolder = `${exportPath}/${task.name}`; - // Remove existing folder - utils.removeDir(taskFolder); - // Recreate it - fs.mkdirSync(taskFolder, function(err){ - if(err){ - console.log(err); - response.send(`ERROR! Can't create directory ${taskFolder}`); - } - }); - - // Write annotations - const streamLabels = utils.iterateOnDB(db, dbkeys.keyForLabels(task.name), false, true); - for await (const labels of streamLabels) { - const data = await getDataDetails(datasetId, labels.data_id, true); - delete data.id; - delete data.dataset_id; - let path = data.path; - path = Array.isArray(path) ? path[0] : path; - path = path.replace(dataset.path, '') - const filename = utils.pathToFilename(path); - - const labelsJson = {...labels, data}; - delete labelsJson.data_id; - - const err = utils.writeJSON(labelsJson, `${taskFolder}/${filename}.json`); - if (err) { - return; - } - } - } - res.send(); - }); + checkAdmin(req, async () => { + if (!req.body.path && !req.body.url) { + return res.status(400).json({ + error: 'wrong_path', + message: 'Invalid path.' + }); + } + if (req.body.path) {//export to local file system + var exportPath = path.join(workspace, req.body.path); + console.log('##### Exporting to ', exportPath); + if (!utils.isSubFolder(workspace, exportPath)) { + return res.status(400).json({ + error: 'wrong_folder', + message: 'Export folder should be a sub folder of the working space.' + }); + } + // If path does not exist create it + if (!fs.existsSync(exportPath)) { + fs.mkdirSync(exportPath, { recursive: true }); + } + } + + const streamTask = utils.iterateOnDB(db, dbkeys.keyForTask(), false, true); + for await (const task of streamTask) { + const spec = await db.get(dbkeys.keyForSpec(task.spec_id)); + delete spec.id; + const dataset = await db.get(dbkeys.keyForDataset(task.dataset_id)); + const datasetId = dataset.id; + delete dataset.id; + const taskJson = { name: task.name, version: annotation_format_version, spec, dataset }; + + // EXPORT task json + if (req.body.path) {//export to local file system + const err = utils.writeJSON(taskJson, `${exportPath}/${task.name}.json`); + if (err) { + return res.status(400).json({ + error: 'cannot_write', + message: `Cannot write json file ${exportPath}/${task.name}.json` + }); + } + } else {//export to destination URL + /// TODO: the task is not exported in Confiance + // var err = ''; + // const url = req.body.url.endsWith('/') ? req.body.url+'_doc' : req.body.url+'/_doc'; + // await fetch(url+`/_doc`, { + // method: 'post', + // headers: { 'Content-Type': 'application/json' }, + // body: JSON.stringify( taskJson ) + // })// send POST request + // .then(res => { + // if (res.statusText=='OK') return res.json(); + // else console.log("KO :\n",res); + // }) + // .then(res => { console.log(res); + // }).catch((e) => { err = e; }); + // if (err) { + // return res.status(400).json({ + // error: 'cannot_write', + // message: `Cannot write json file '${task.name}.json'.\n\nERROR while calling ELASTIC:${err}` + // }); + // } + } + + if (req.body.path) { + // Write annotations for each task in a specific folder + var taskFolder = `${exportPath}/${task.name}`; + // Remove existing folder + utils.removeDir(taskFolder); + // Recreate it + fs.mkdirSync(taskFolder, function (err) { + if (err) { + console.log(err); + return res.status(400).json({ + error: 'cannot_create', + message: `ERROR! Can't create directory ${taskFolder}` + }); + } + }); + } + + // Write annotations + const streamLabels = utils.iterateOnDB(db, dbkeys.keyForLabels(task.name), false, true); + for await (const labels of streamLabels) { + resultData = await db.get(dbkeys.keyForResult(task.name, labels.data_id));//get the status for this data + const data = await getDataDetails(datasetId, labels.data_id, true); + delete data.id; + delete data.dataset_id; + delete data.thumbnail; + data.status = resultData.status;//add the status + let path = data.path; + path = Array.isArray(path) ? path[0] : path; + path = path.replace(dataset.path, '') + const filename = utils.pathToFilename(path); + + const labelsJson = { ...labels, data }; + + // EXPORT task json + if (req.body.path) {//export to local file system + const err = utils.writeJSON(labelsJson, `${taskFolder}/${filename}.json`); + if (err) { + return res.status(400).json({ + error: 'cannot_write', + message: `Cannot write json file ${taskFolder}/${filename}.json` + }); + } + } else {//export to destination URL + var err = ''; + const url = req.body.url.endsWith('/') ? req.body.url+'_doc' : req.body.url+'/_doc'; + + await fetch(url, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify( labelsJson ) + })// send POST request + .then(res => { + if (res.ok) return res.json(); + else throw new Error(res);//we have to trow ourself because fetch only throw on network errors, not on 4xx or 5xx errors + }).catch((e) => { err = e; }); + if (err) { + return res.status(400).json({ + error: 'cannot_write', + message: `Cannot write json file '${filename}.json'.\n\nERROR while calling ELASTIC:${err}` + }); + } + } + } + } + res.send(); + }); } /**