diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml
index 30265baa..4f7c9998 100644
--- a/.github/workflows/build_job.yml
+++ b/.github/workflows/build_job.yml
@@ -19,7 +19,7 @@ jobs:
steps:
-
name: Checkout repository
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
with:
fetch-depth: 0
diff --git a/.github/workflows/test_job.yml b/.github/workflows/test_job.yml
index 7e99d391..642312bc 100644
--- a/.github/workflows/test_job.yml
+++ b/.github/workflows/test_job.yml
@@ -3,36 +3,28 @@ run-name: ${{ github.actor }} is doing some smoke tests
on: [push, pull_request, workflow_call]
jobs:
- lint:
+ Smoke-tests:
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
submodules: 'true'
- name: Set up services
run: docker-compose up -d --remove-orphans
- name: Create conda env
- uses: conda-incubator/setup-miniconda@v2
+ # https://github.com/marketplace/actions/setup-micromamba
+ uses: mamba-org/setup-micromamba@v1
with:
- activate-environment: freva-web
+ environment-name: freva-web
environment-file: conda-env.yml
- python-version: "3.12"
- auto-activate-base: false
- - name: Run checks in python
- run: make tests
- - name: Lint python
- run: make lint
- - name: Prepare node
- run: npm install
- - name: Lint js formatting
- run: npm run lint-format
- - name: Lint javascript
- run: npm run lint
- - name: Test building the prod system
- run: npm run build-production
- - name: Test building the dev system
- run: npm run build
+ cache-environment: false
+ cache-downloads: false
+ init-shell: bash
+ - name: Lint js code and python
+ run: micromamba run -n freva-web make lint
+ - name: Run build checks for js and python smoke tests
+ run: micromamba run -n freva-web make tests
diff --git a/.gitignore b/.gitignore
index 76339d08..f712de07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,10 @@
*.pyc
*.db
+*.log
virt
static_root/img/favicon.svg
freva_web.conf
+docker/databrowser_api_config.toml
db
node_modules
venv/*
@@ -22,3 +24,5 @@ activate_web
static/*
assets/bundles/*
webpack-stats.json
+base/migrations/*
+
diff --git a/.gitmodules b/.gitmodules
index a6e09870..69e067c3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,4 @@
[submodule "docker/config"]
path = docker/config
url = https://github.com/FREVA-CLINT/freva-service-config.git
+ branch = main
diff --git a/Makefile b/Makefile
index 4048377b..c5c1c1f5 100644
--- a/Makefile
+++ b/Makefile
@@ -2,17 +2,62 @@
export EVALUATION_SYSTEM_CONFIG_FILE := $(PWD)/docker/local-eval-system.conf
export EVALUATION_SYSTEM_DRS_CONFIG_FILE := $(PWD)/docker/drs_config.toml
-run:
- python manage.py runserver
+export DJANGO_SUPERUSER_PASSWORD := secret
+export DEV_MODE := 1
+
+.PHONY: all run runserver runfrontend stopserver stopfrontend stop setup
+
+all: setup runserver runfrontend
+ @echo "All services are running in the background."
dummy-data:
docker/dummy_plugin_runs.sh
- python3 docker/solr/ingest_dummy_data.py
-lint:
+
+setup-django:
+ python manage.py makemigrations base
+ python manage.py migrate --fake-initial
+ python manage.py migrate --fake contenttypes
+ python manage.py createsuperuser --noinput --username admin --email foo@bar.com.au || echo
+
+setup-node:
+ npm install
+
+runserver:
+ @echo "Starting Django development server..."
+ python manage.py runserver > runserver.log 2>&1 &
+ @echo "Django development server is running..."
+ @echo "To watch the Django server logs, run 'tail -f runserver.log'"
+
+runfrontend:
+ @echo "Starting npm development server..."
+ npm run dev > npm.log 2>&1 &
+ @echo "npm development server is running..."
+ @echo "To watch the npm logs, run 'tail -f npm.log'"
+
+stopserver:
+ ps aux | grep '[m]anage.py runserver' | awk '{print $$2}' | xargs -r kill
+ echo "Stopped Django development server..." > runserver.log
+
+stopfrontend:
+ pkill -f "npm run dev"
+ echo "Stopped npm development server..." > npm.log
+
+stop: stopserver stopfrontend
+ @echo "All services have been stopped."
+
+setup: setup-node setup-django dummy-data
+
+run: runfrontend runserver
+
+lint: setup-node
+ npm run lint-format
+ npm run lint
isort -c --profile black -t py312 .
-tests:
+tests: setup-node
+ npm run build-production
+ npm run build
rm -rf node_modules
pytest -vv $(PWD) tests/
diff --git a/README.md b/README.md
index 27c2f0e5..e75ceea4 100644
--- a/README.md
+++ b/README.md
@@ -25,8 +25,16 @@ conda env create -f conda-env.yml
source .envrc
```
-The web ui will need a connection to a solr and mariadb service.
-This services can be deployed using
+> ``📝`` If conda has issues solving dependencies you can install and use
+ [mamba](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html)
+ instead of anaconda. This is recommended because the dependency solvers
+ that ship with mamba are usually much faster than those conda uses.
+
+### Additional services running on docker
+
+The web ui will need a connection to a solr,
+[freva-databrowser](https://github.com/FREVA-CLINT/databrowserAPI/) and
+mariadb service. This services can be deployed using
[`docker-compose`](https://docs.docker.com/compose/install/).
```console
@@ -43,7 +51,7 @@ docker-compose down
There are some rudimentary tests that check the integration of `django` and the
`nodejs` building process. Assuming you have followed steps mentioned above and
-created a `freva-dev` cona miniconda environment you can run the tests after
+created a `freva-dev` conda miniconda environment you can run the tests after
activating this environment:
```console
@@ -51,61 +59,105 @@ conda activate freva-web
python -m pytest -vv tests
```
-## Django deployment
-
-You can check if django is working and correctly configured by:
-
-```console
-python manage.py check
-```
+## Using GNU `make`:
+We have created a Makefile that sets up a development version of the web. You
+can use:
-If checks are passing issue the following command
+- To *setup* / *initialise* the nodejs and django servers use:
+ ```console
+ make setup
+ ```
+- To *run* node and django servers use:
+ ```console
+ make run
+ ```
+- To use both `setup` and `run` command use just use make:
+ ```console
+ make
+ ```
+- To stop the servers use:
+ ```console
+ make stop
+ ```
-```console
-python manage.py makemigrations base
-python manage.py migrate --fake-initial
-python manage.py migrate --fake contenttypes
-python manage.py createsuperuser
-```
+The django and npm development servers will write output into `runserver.log` and
+`npm.log`. You can observe the output of the processes using `tail -f` or something
+similar.
-The `--fake-inital` flag tells `django` not to create the already existing
-database tables.
+> ``📝`` You need a Node version of at least 16.5 along a npm version of 8.19
-## Running the server in dev mode
+# The Production container
+This section only briefly describes the docker image that is automatically
+created (see the [next section](#create-a-new-web-release) on how to trigger a
+build of the image) and is not meant for actual application. Please install and
+use the [freva-deployment package](https://pypi.org/project/freva-deployment/)
+to deploy the web app in production mode.
-A development server can be set using the following command:
+A pre-build image of the web app is available via:
```console
-python manage.py runserver [port_number]
+docker pull ghcr.io/freva-clint/freva-web:latest
```
-Default port is 8000. If an application is already running on that port you
-can change the port number with help of a command line argument
-
-## Building the JS application :
+When running in production mode you should set the following container
+environment variables:
-**Please note:** You need a Node version of at least 16.5 along a npm version of 8.19
+- ``EVALUATION_SYSTEM_CONFIG_FILE`` : Path to the freva config file
+- ``LDAP_USER_DN``: the Distinguished Name within ldap for example
+ `uid=jdoe,ou=users,dc=example,dc=com`.
+- ``LDAP_USER_PW``: password for the LDAP server connection.
+- ``DJANGO_SUPERUSER_PASSWORD``: the super user password for the django app.
-Install dependencies:
+The web app app is running on port 8000, hence you want to publish this port
+via the `-p` flag. Ideally the path to the `$EVALUATION_SYSTEM_CONFIG_FILE`
+should be mounted into the container as a volume.
-```console
+Since static files are served by the http web server and not the django web app
+you have to add the location of the static files (e.g. `/srv/static`) as a
+volume into the container to the `/opt/freva_web/static` location.
+On startup of the container django app will create all static files that will
+then be available through `/srv/static` on the docker host.
-npm install
-
-```
-
-Build project:
+All together a minimal working example looks like this:
```console
-npm run build
-npm run build-production # optimized production build
+docker run -it -e EVALUATION_SYSTEM_CONFIG_FILE=/work/freva/evaluation_system.conf \
+ -e LDAP_USER_DN='uid=jdoe,ou=users,dc=example,dc=com' \
+ -e LDAP_USER_PW='secret' \
+ -e DJANGO_SUPERUSER_PASSWORD='more_secret' \
+ -v /work/freva:/work/freva:z \
+ -v /srv/static:/opt/freva_web/static:z \
+ -p 8000:8000 \
+ ghcr.io/freva-clint/freva-web:latest
```
+The web app is then available via port 8000 on the host system.
-Development:
+## Making the web app available on a web server.
+To be able to access the web app through a normal http web server you will need
+to setup a reverse proxy on your http web server to port 8000. Refer to the
+reverse proxy settings for your web server. Here is a minimal example for
+apache httpd (using the example from above where the static files are located
+in `/srv/static` on the docker host):
-```console
-npm run dev # starts webpack-dev-server including hot-reloading
```
+Listen 80
+LoadModule proxy_html_module modules/mod_proxy_html.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+
freva databrowser + {props.selectedFlavour !== constants.DEFAULT_FLAVOUR + ? ` --flavour ${props.selectedFlavour} ` + : ""} {props.minDate && (time= @@ -68,7 +76,7 @@ function DataBrowserCommandImpl(props) { return ( {" "} - {key}={value} + {props.facetMapping[key]}={value} ); })} @@ -86,10 +94,17 @@ function DataBrowserCommandImpl(props) { } function getFullPythonCommand(dateSelectorToCli) { - const args = Object.keys(selectedFacets).map((key) => { - const value = selectedFacets[key]; - return `${key}="${value}"`; - }); + let args = []; + if (props.selectedFlavour !== constants.DEFAULT_FLAVOUR) { + args.push(`flavour=${props.selectedFlavour}`); + } + args = [ + ...args, + ...Object.keys(selectedFacets).map((key) => { + const value = selectedFacets[key]; + return `${props.facetMapping[key]}="${value}"`; + }), + ]; props.minDate && args.push(`time="${props.minDate} to ${props.maxDate}"`); props.minDate && args.push(`time_select="${dateSelectorToCli}"`); return `freva.databrowser(${args.join(", ")})`.trimEnd(); @@ -163,6 +178,8 @@ DataBrowserCommandImpl.propTypes = { dateSelector: PropTypes.string, minDate: PropTypes.string, maxDate: PropTypes.string, + selectedFlavour: PropTypes.string, + facetMapping: PropTypes.object, }; const mapStateToProps = (state) => ({ @@ -170,6 +187,8 @@ const mapStateToProps = (state) => ({ dateSelector: state.databrowserReducer.dateSelector, minDate: state.databrowserReducer.minDate, maxDate: state.databrowserReducer.maxDate, + facetMapping: state.databrowserReducer.facetMapping, + selectedFlavour: state.databrowserReducer.selectedFlavour, }); export default connect(mapStateToProps)(DataBrowserCommandImpl); diff --git a/assets/js/Containers/Databrowser/FacetPanel.js b/assets/js/Containers/Databrowser/FacetPanel.js new file mode 100644 index 00000000..2bd26890 --- /dev/null +++ b/assets/js/Containers/Databrowser/FacetPanel.js @@ -0,0 +1,74 @@ +import React from "react"; +import PropTypes from "prop-types"; + +import { Badge } from "react-bootstrap"; + +import { initCap, underscoreToBlank } from "../../utils"; +import AccordionItemBody from "../../Components/AccordionItemBody"; +import OwnPanel from "../../Components/OwnPanel"; + +/* +This class represents a single Dropdown Menu for a single facet including the title of the facet +*/ +export function FacetPanel({ + value, + keyVar, + metadata, + selectedFacets, + facetMapping, + clickFacet, + isFacetCentered, +}) { + const isFacetSelected = !!selectedFacets[keyVar]; + const facetTitle = initCap(underscoreToBlank(facetMapping[keyVar] ?? keyVar)); + let panelHeader; + if (isFacetSelected) { + panelHeader = ( + + {facetTitle}: {selectedFacets[keyVar]} + + ); + } else if (value.length === 2) { + panelHeader = ( + + {facetTitle}: {value[0]} + + ); + } else { + const numberOfValues = value.length / 2; + panelHeader = ( + + {facetTitle} ++ {numberOfValues.toLocaleString("en-US")} + + + ); + } + return ( +clickFacet(keyVar) : null} + > + + ); +} + +FacetPanel.propTypes = { + value: PropTypes.array, + keyVar: PropTypes.string, + metadata: PropTypes.object, + facetMapping: PropTypes.object, + selectedFacets: PropTypes.object, + isFacetCentered: PropTypes.bool, + clickFacet: PropTypes.func.isRequired, +}; diff --git a/assets/js/Containers/Databrowser/FilesPanel.js b/assets/js/Containers/Databrowser/FilesPanel.js index d581566f..bfaf0007 100644 --- a/assets/js/Containers/Databrowser/FilesPanel.js +++ b/assets/js/Containers/Databrowser/FilesPanel.js @@ -1,14 +1,20 @@ import React, { useState } from "react"; import PropTypes from "prop-types"; import { connect } from "react-redux"; -import { Tooltip, OverlayTrigger, Button, Badge } from "react-bootstrap"; +import { Tooltip, OverlayTrigger, Button } from "react-bootstrap"; +import { withRouter } from "react-router"; import { FaInfoCircle } from "react-icons/fa"; +import queryString from "query-string"; + import NcdumpDialog, { NcDumpDialogState } from "../../Components/NcdumpDialog"; import CircularSpinner from "../../Components/Spinner"; import { getCookie } from "../../utils"; +import Pagination from "../../Components/Pagination"; + +import { BATCH_SIZE } from "./constants"; function FilesPanelImpl(props) { const { files, numFiles, fileLoading } = props.databrowser; @@ -20,6 +26,15 @@ function FilesPanelImpl(props) { error: null, }); + function setPageOffset(offset) { + const currentLocation = props.location.pathname; + const query = queryString.stringify({ + ...props.location.query, + start: (offset - 1) * BATCH_SIZE, + }); + props.router.push(currentLocation + "?" + query); + } + function loadNcdump(fn, pw) { const url = "/api/solr/ncdump/"; setNcDump({ ...ncdump, status: NcDumpDialogState.LOADING }); @@ -40,7 +55,6 @@ function FilesPanelImpl(props) { if (!resp.ok) { /* eslint-disable */ return resp.json().then((json) => { - console.log(resp.statusText); if (json.error_msg) { throw new Error(json.error_msg); } else { @@ -67,12 +81,23 @@ function FilesPanelImpl(props) { } const [filename, setFilename] = useState(null); + return (+ -- Files -
+ +{numFiles.toLocaleString("en-US")} -+ Files +
++++ ({ error: state.appReducer.error, }); -export default connect(mapStateToProps)(FilesPanelImpl); +export default withRouter(connect(mapStateToProps)(FilesPanelImpl)); diff --git a/assets/js/Containers/Databrowser/FacetDropdown.js b/assets/js/Containers/Databrowser/MetaFacet.js similarity index 67% rename from assets/js/Containers/Databrowser/FacetDropdown.js rename to assets/js/Containers/Databrowser/MetaFacet.js index 18ccc49f..5529064a 100644 --- a/assets/js/Containers/Databrowser/FacetDropdown.js +++ b/assets/js/Containers/Databrowser/MetaFacet.js @@ -7,7 +7,7 @@ import { Badge } from "react-bootstrap"; import Select from "../../Components/Select"; import { initCap, underscoreToBlank } from "../../utils"; -function FacetDropdownImpl(props) { +function MetaFacetImpl(props) { const options = []; function constructValueName(category, value) { @@ -15,9 +15,17 @@ function FacetDropdownImpl(props) { const valueInfos = additionalInfo && additionalInfo[value]; return value + (valueInfos ? " " + valueInfos : ""); } + const primaryFacetsSets = new Set(props.primaryFacets); + const sortedFacets = [ + ...props.primaryFacets, + ...Object.keys(props.facetMapping).filter((x) => !primaryFacetsSets.has(x)), + ]; - for (const f in props.facets) { + for (const f of sortedFacets) { const facet = props.facets[f]; + if (!facet) { + continue; + } for (let i = 0; i < facet.length; i = i + 2) { options.push({ value: constructValueName(f, facet[i]), @@ -26,10 +34,14 @@ function FacetDropdownImpl(props) { label: (
), }); @@ -54,18 +66,22 @@ function FacetDropdownImpl(props) { ); } -FacetDropdownImpl.propTypes = { +MetaFacetImpl.propTypes = { className: PropTypes.string, clickFacet: PropTypes.func.isRequired, facets: PropTypes.object, metadata: PropTypes.object, selectedFacets: PropTypes.object, + primaryFacets: PropTypes.array, + facetMapping: PropTypes.object, }; const mapStateToProps = (state) => ({ facets: state.databrowserReducer.facets, metadata: state.databrowserReducer.metadata, + primaryFacets: state.databrowserReducer.primaryFacets, + facetMapping: state.databrowserReducer.facetMapping, selectedFacets: state.databrowserReducer.selectedFacets, }); -export default connect(mapStateToProps)(FacetDropdownImpl); +export default connect(mapStateToProps)(MetaFacetImpl); diff --git a/assets/js/Containers/Databrowser/actions.js b/assets/js/Containers/Databrowser/actions.js index a3ec8efe..fe507674 100644 --- a/assets/js/Containers/Databrowser/actions.js +++ b/assets/js/Containers/Databrowser/actions.js @@ -1,12 +1,11 @@ import fetch from "isomorphic-fetch"; -import queryString from "query-string"; - import { getCookie } from "../../utils"; import * as globalStateConstants from "../App/constants"; import * as constants from "./constants"; +import { prepareSearchParams } from "./utils"; export const updateFacetSelection = (queryObject) => (dispatch) => { dispatch({ @@ -20,9 +19,33 @@ export const setMetadata = (metadata) => ({ metadata, }); -export const loadFacets = (location) => (dispatch) => { - dispatch({ type: constants.SET_FACET_LOADING }); - return fetchResults(dispatch, location, "facet=*", constants.LOAD_FACETS); +export const setFlavours = () => (dispatch) => { + return fetch("/api/databrowser/overview", { + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + Accept: "application/json", + "Content-Type": "application/json", + }, + }) + .then((response) => { + if (response.status >= 400) { + throw new Error(response.statusText); + } + return response.json(); + }) + .then((json) => { + return dispatch({ + type: constants.SET_FLAVOURS, + payload: json, + }); + }) + .catch(() => { + return dispatch({ + type: globalStateConstants.SET_ERROR, + payload: "Internal Error: Could not load flavours", + }); + }); }; export const loadFiles = (location) => (dispatch) => { @@ -30,37 +53,14 @@ export const loadFiles = (location) => (dispatch) => { return fetchResults( dispatch, location, - "start=0&rows=100", + `max-results=${constants.BATCH_SIZE}&translate=false`, constants.LOAD_FILES ); }; function fetchResults(dispatch, location, additionalParams, actionType) { - // const { selectedFacets, dateSelector, minDate, maxDate } = getState().databrowserReducer; - let params = ""; - if (location) { - const queryObject = location.query; - const { - dateSelector: ignore1, - minDate: ignore2, - maxDate: ignore3, - ...facets - } = location.query; - - params = queryString.stringify(facets); - - if (params) { - params = "&" + params; - } - - const isDateSelected = !!queryObject.minDate; - if (isDateSelected) { - params += `&time_select=${queryObject.dateSelector}&time=${queryObject.minDate} TO ${queryObject.maxDate}`; - } - } - - const url = `/solr/solr-search/?${additionalParams}${params}`; - + const searchParams = prepareSearchParams(location, additionalParams); + const url = `/api/databrowser/extended_search/${searchParams}`; return fetch(url, { credentials: "same-origin", headers: { @@ -78,7 +78,11 @@ function fetchResults(dispatch, location, additionalParams, actionType) { .then((json) => { return dispatch({ type: actionType, - payload: json, + payload: { + ...json, + start: location.query.start ?? 0, + flavour: location.query.flavour ?? constants.DEFAULT_FLAVOUR, + }, }); }) .catch(() => { diff --git a/assets/js/Containers/Databrowser/constants.js b/assets/js/Containers/Databrowser/constants.js index 53db9b2a..c488207b 100644 --- a/assets/js/Containers/Databrowser/constants.js +++ b/assets/js/Containers/Databrowser/constants.js @@ -6,7 +6,8 @@ export const UPDATE_FACET_SELECTION = "UPDATE_FACET_SELECTION"; export const SET_METADATA = "SET_METADATA"; -export const SET_FACET_LOADING = "SET_FACET_LOADING"; +export const SET_FLAVOURS = "SET_FLAVOURS"; + export const SET_FILE_LOADING = "SET_FILE_LOADING"; export const LOAD_NCDUMP = "LOAD_NCDUMP"; @@ -17,3 +18,12 @@ export const RESET_NCDUMP = "RESET_NCDUMP"; export const TIME_RANGE_FLEXIBLE = "flexible"; export const TIME_RANGE_STRICT = "strict"; export const TIME_RANGE_FILE = "file"; + +export const DEFAULT_FLAVOUR = "freva"; +export const BATCH_SIZE = 100; +export const ViewTypes = { + RESULT_CENTERED: "RESULT_CENTERED", + FACET_CENTERED: "FACET_CENTERED", +}; + +export const INTAKE_MAXIMUM = 100_000; diff --git a/assets/js/Containers/Databrowser/index.js b/assets/js/Containers/Databrowser/index.js index 27070ed5..02787116 100644 --- a/assets/js/Containers/Databrowser/index.js +++ b/assets/js/Containers/Databrowser/index.js @@ -7,45 +7,56 @@ import { Col, Button, Alert, - Badge, OverlayTrigger, Tooltip, + Form, + Collapse, } from "react-bootstrap"; -import { FaAlignJustify, FaList, FaTimes } from "react-icons/fa"; +import { + FaAlignJustify, + FaFileExport, + FaList, + FaMinusSquare, + FaPlusSquare, + FaTimes, +} from "react-icons/fa"; import queryString from "query-string"; import { withRouter } from "react-router"; -import AccordionItemBody from "../../Components/AccordionItemBody"; import OwnPanel from "../../Components/OwnPanel"; import Spinner from "../../Components/Spinner"; import { initCap, underscoreToBlank } from "../../utils"; import { - loadFacets, setMetadata, loadFiles, updateFacetSelection, + setFlavours, } from "./actions"; import TimeRangeSelector from "./TimeRangeSelector"; import FilesPanel from "./FilesPanel"; import DataBrowserCommand from "./DataBrowserCommand"; -import FacetDropdown from "./FacetDropdown"; - -const ViewTypes = { - RESULT_CENTERED: "RESULT_CENTERED", - FACET_CENTERED: "FACET_CENTERED", -}; +import FacetDropdown from "./MetaFacet"; +import { ViewTypes, DEFAULT_FLAVOUR, INTAKE_MAXIMUM } from "./constants"; +import { FacetPanel } from "./FacetPanel"; +import { prepareSearchParams } from "./utils"; class Databrowser extends React.Component { constructor(props) { super(props); this.clickFacet = this.clickFacet.bind(this); this.renderFacetBadges = this.renderFacetBadges.bind(this); - this.state = { viewPort: ViewTypes.RESULT_CENTERED }; + const firstViewPort = + localStorage.FrevaDatabrowserViewPort ?? ViewTypes.RESULT_CENTERED; + localStorage.FrevaDatabrowserViewPort = firstViewPort; + this.state = { + viewPort: firstViewPort, + additionalFacetsVisible: false, + }; } /** @@ -53,7 +64,7 @@ class Databrowser extends React.Component { * Also load the metadata.js script */ componentDidMount() { - this.props.dispatch(loadFacets(this.props.location)); + this.props.dispatch(setFlavours()); this.props.dispatch(loadFiles(this.props.location)); this.props.dispatch(updateFacetSelection(this.props.location.query)); const script = document.createElement("script"); @@ -80,7 +91,6 @@ class Databrowser extends React.Component { componentDidUpdate(prevProps) { if (prevProps.location.search !== this.props.location.search) { - this.props.dispatch(loadFacets(this.props.location)); this.props.dispatch(loadFiles(this.props.location)); this.props.dispatch(updateFacetSelection(this.props.location.query)); } @@ -94,13 +104,14 @@ class Databrowser extends React.Component { // delete const { [category]: toRemove, ...queryObject } = this.props.location.query; - const query = queryString.stringify(queryObject); + const query = queryString.stringify({ ...queryObject, start: 0 }); this.props.router.push(currentLocation + "?" + query); return; } const query = queryString.stringify({ ...this.props.location.query, [category]: value, + start: 0, }); if (query) { this.props.router.push(currentLocation + "?" + query); @@ -109,64 +120,69 @@ class Databrowser extends React.Component { } } - // dropFacet(category) { - // const currentLocation = this.props.location.pathname; - // const { [category]: toRemove, ...queryObject } = this.props.location.query; - // const query = queryString.stringify(queryObject); - // this.props.router.push(currentLocation + "?" + query); - // } + clickFlavour(value = DEFAULT_FLAVOUR) { + const currentLocation = this.props.location.pathname; + const query = queryString.stringify({ + ...this.props.location.query, + flavour: value, + }); + if (query) { + this.props.router.push(currentLocation + "?" + query); + } else { + this.props.router.push(currentLocation); + } + } /** * Loop all facets and render the panels */ renderFacetPanels() { - const { facets, selectedFacets, metadata } = this.props.databrowser; - // const { dispatch } = this.props; + const { facets, primaryFacets, selectedFacets, facetMapping, metadata } = + this.props.databrowser; - return Object.keys(facets).map((key) => { + return primaryFacets.map((key) => { const value = facets[key]; - let panelHeader; - const isFacetSelected = !!selectedFacets[key]; - if (isFacetSelected) { - panelHeader = ( - - {initCap(underscoreToBlank(key))}:{" "} - {selectedFacets[key]} - - ); - } else if (value.length === 2) { - panelHeader = ( - - {initCap(underscoreToBlank(key))}: {value[0]} - - ); - } else { - const numberOfValues = value.length / 2; - panelHeader = ( - - {initCap(underscoreToBlank(key))} ---{initCap(underscoreToBlank(f))} {" "} ++ {initCap(underscoreToBlank(props.facetMapping[f]))} + {" "} {facet[i]}{facet[i + 1]} ++ {parseInt(facet[i + 1]).toLocaleString("en-US")} + - {numberOfValues} - - - ); - } + if (!value) return undefined; return ( -this.clickFacet(key) : null} - > - - ); - }); + ); + }); } dropTimeSelection() { @@ -185,24 +201,30 @@ class Databrowser extends React.Component { } } + createIntakeLink() { + return ( + "/api/databrowser/intake_catalogue/" + + prepareSearchParams(this.props.location, "translate=false") + ); + } + renderTimeSelectionPanel() { - const key = "time_range"; const { dateSelector, minDate, maxDate } = this.props.databrowser; const isDateSelected = !!minDate; const title = isDateSelected ? ( - Time Range: + Time : {dateSelector}: {minDate} to {maxDate} ) : ( - Time Range + Time ); return (+ ); + }); + } + + renderAdditionalFacets() { + const { facets, primaryFacets, selectedFacets, facetMapping, metadata } = + this.props.databrowser; + const primaryFacetsSet = new Set(primaryFacets); + + return Object.keys(facetMapping) + .filter((x) => { + return !primaryFacetsSet.has(x); + }) + .map((key) => { + const value = facets[key]; + if (!value) return undefined; + return ( + - this.dropTimeSelection() : null} > @@ -245,8 +267,8 @@ class Databrowser extends React.Component { }} key={"selected-" + x + this.props.databrowser.selectedFacets[x]} > - {initCap(underscoreToBlank(x))}:{" "} - {this.props.databrowser.selectedFacets[x]} + {initCap(underscoreToBlank(this.props.databrowser.facetMapping[x]))} + : {this.props.databrowser.selectedFacets[x]} ); @@ -292,7 +314,9 @@ class Databrowser extends React.Component { } const facetPanels = this.renderFacetPanels(); + const additionalFacetPanels = this.renderAdditionalFacets(); const isFacetCentered = this.state.viewPort === ViewTypes.FACET_CENTERED; + const flavour = this.props.location.query.flavour; return ( @@ -303,7 +327,52 @@ class Databrowser extends React.Component {
)} - ++{ + this.clickFlavour(x.target.value); + }} + > + {this.props.databrowser.flavours.map((x) => { + return ; + })} + + {this.props.databrowser.numFiles > INTAKE_MAXIMUM ? ( ++ Please narrow down your search to a maximum of 100,000 + results in order to enable Intake exports + + } + > + + + + + ) : ( + + +Intake catalogue + + + )} Change view with facets in focus} > @@ -312,7 +381,13 @@ class Databrowser extends React.Component { variant="outline-secondary" active={isFacetCentered} onClick={() => - this.setState({ viewPort: ViewTypes.FACET_CENTERED }) + this.setState( + { viewPort: ViewTypes.FACET_CENTERED }, + () => { + localStorage.FrevaDatabrowserViewPort = + this.state.viewPort; + } + ) } > @@ -325,7 +400,13 @@ class Databrowser extends React.Component { variant="outline-secondary" active={!isFacetCentered} onClick={() => - this.setState({ viewPort: ViewTypes.RESULT_CENTERED }) + this.setState( + { viewPort: ViewTypes.RESULT_CENTERED }, + () => { + localStorage.FrevaDatabrowserViewPort = + this.state.viewPort; + } + ) } > @@ -342,6 +423,35 @@ class Databrowser extends React.Component { {facetPanels} {this.renderTimeSelectionPanel()} + + + {additionalFacetPanels}+@@ -358,10 +468,13 @@ Databrowser.propTypes = { databrowser: PropTypes.shape({ facets: PropTypes.object, files: PropTypes.array, + flavours: PropTypes.array, fileLoading: PropTypes.bool, facetLoading: PropTypes.bool, + facetMapping: PropTypes.object, numFiles: PropTypes.number, selectedFacets: PropTypes.object, + primaryFacets: PropTypes.array, metadata: PropTypes.object, dateSelector: PropTypes.string, minDate: PropTypes.string, diff --git a/assets/js/Containers/Databrowser/reducers.js b/assets/js/Containers/Databrowser/reducers.js index 86eb5081..69b33fde 100644 --- a/assets/js/Containers/Databrowser/reducers.js +++ b/assets/js/Containers/Databrowser/reducers.js @@ -4,6 +4,7 @@ const databrowserInitialState = { facets: null, files: [], numFiles: 0, + start: 0, selectedFacets: {}, minDate: "", maxDate: "", @@ -11,12 +12,12 @@ const databrowserInitialState = { metadata: {}, facetLoading: false, fileLoading: false, + flavours: ["freva"], + selectedFlavour: constants.DEFAULT_FLAVOUR, }; export const databrowserReducer = (state = databrowserInitialState, action) => { switch (action.type) { - case constants.SET_FACET_LOADING: - return { ...state, facetLoading: true }; case constants.SET_FILE_LOADING: return { ...state, fileLoading: true }; case constants.LOAD_FACETS: @@ -27,17 +28,8 @@ export const databrowserReducer = (state = databrowserInitialState, action) => { facetLoading: false, }; case constants.UPDATE_FACET_SELECTION: { - const { minDate, maxDate, dateSelector, ...queryObject } = + const { minDate, maxDate, dateSelector, start, flavour, ...queryObject } = action.queryObject; - // let newObject = {} - // if (state.facets) { - // Object.keys(state.facets).forEach(key => { - // if (key in queryObject) { - // newObject = { ...newObject, [key]: queryObject[key] }; - // } - // }); - // } - // let dateInfo = {}; let myMinDate = minDate; let myMaxDate = maxDate; let myDateSelector = dateSelector; @@ -49,19 +41,28 @@ export const databrowserReducer = (state = databrowserInitialState, action) => { return { ...state, selectedFacets: { ...queryObject }, + start: start !== undefined ? parseInt(start) : 0, dateSelector: myDateSelector, minDate: myMinDate, maxDate: myMaxDate, + selectedFlavour: flavour || databrowserInitialState.selectedFlavour, }; } case constants.SET_METADATA: return { ...state, metadata: action.metadata }; + case constants.SET_FLAVOURS: + return { ...state, flavours: action.payload.flavours }; case constants.LOAD_FILES: return { ...state, - files: action.payload.data, - numFiles: action.payload.metadata.numFound, + facets: action.payload.facets, + primaryFacets: action.payload.primary_facets, + facetMapping: action.payload.facet_mapping, + files: action.payload.search_results.map((x) => x.file), + numFiles: action.payload.total_count, fileLoading: false, + start: parseInt(action.payload.start), + selectedFlavour: action.payload.flavour, }; default: return state; diff --git a/assets/js/Containers/Databrowser/utils.js b/assets/js/Containers/Databrowser/utils.js new file mode 100644 index 00000000..d26e557a --- /dev/null +++ b/assets/js/Containers/Databrowser/utils.js @@ -0,0 +1,24 @@ +import queryString from "query-string"; + +import * as constants from "./constants"; + +export function prepareSearchParams(location, additionalParams = "") { + let params = ""; + let flavourValue = constants.DEFAULT_FLAVOUR; + if (location) { + const queryObject = location.query; + const { dateSelector, minDate, maxDate, flavour, ...facets } = queryObject; + flavourValue = flavour ?? flavourValue; + params = queryString.stringify(facets); + + if (params && additionalParams) { + params = "&" + params; + } + + const isDateSelected = !!minDate; + if (isDateSelected) { + params += `&time_select=${dateSelector}&time=${minDate} TO ${maxDate}`; + } + } + return `${flavourValue}/file?${additionalParams}${params}`; +} diff --git a/assets/js/Containers/PluginList/index.js b/assets/js/Containers/PluginList/index.js index b5445d66..31725d19 100644 --- a/assets/js/Containers/PluginList/index.js +++ b/assets/js/Containers/PluginList/index.js @@ -2,6 +2,7 @@ import React from "react"; import PropTypes from "prop-types"; import { browserHistory } from "react-router"; import { connect } from "react-redux"; +import { FaTimes } from "react-icons/fa"; import { Row, Col, @@ -186,7 +187,10 @@ class PluginList extends React.Component { children = {error}; } - if (!pluginsLoaded && !this.props.pluginList.errorMessage) { + if ( + (!pluginsLoaded && !this.props.pluginList.errorMessage) || + !currentUser + ) { return; } if (this.props.pluginList.errorMessage) { @@ -204,7 +208,7 @@ class PluginList extends React.Component { return ( - +
|
Plugins
@@ -227,14 +231,14 @@ class PluginList extends React.Component {-
+ {Object.keys(categories).map((key) => { return this.renderPluginBlock(filteredPlugins, key); })} - + - -Categories: ++-+ Categories + {categoriesFilter.length > 0 && ( + + + + )} + {Object.keys(categories).map((key) => { return this.renderCategoryCheckbox( @@ -255,17 +274,40 @@ class PluginList extends React.Component { })}-Tags: +++ Tags + {tagsFilter.length > 0 && ( + + + + )} + {tags.map((tag) => { + const variant = _.includes(tagsFilter, tag) + ? "success" + : "secondary"; + const disabled = !filteredPlugins.some((x) => { + return _.includes(x[1].tags, tag); + }); return ({% endif %}++{# Render a Bootstrap sendmail dialog #} diff --git a/templates/history/templatetags/mailfield.html b/templates/history/templatetags/mailfield.html index 5030ebcd..c9ed018e 100644 --- a/templates/history/templatetags/mailfield.html +++ b/templates/history/templatetags/mailfield.html @@ -1,17 +1,28 @@ {% load resulttags %} - + diff --git a/templates/history/templatetags/sendmail_dialog.html b/templates/history/templatetags/sendmail_dialog.html index 87bd7291..ce0dc045 100644 --- a/templates/history/templatetags/sendmail_dialog.html +++ b/templates/history/templatetags/sendmail_dialog.html @@ -2,24 +2,26 @@ {% load dialogtags %} {% load settingstags %} - -- - - ++ +- + + + +