diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 30265baa..4f7c9998 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/test_job.yml b/.github/workflows/test_job.yml index 7e99d391..642312bc 100644 --- a/.github/workflows/test_job.yml +++ b/.github/workflows/test_job.yml @@ -3,36 +3,28 @@ run-name: ${{ github.actor }} is doing some smoke tests on: [push, pull_request, workflow_call] jobs: - lint: + Smoke-tests: runs-on: ubuntu-latest defaults: run: shell: bash -l {0} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: 'true' - name: Set up services run: docker-compose up -d --remove-orphans - name: Create conda env - uses: conda-incubator/setup-miniconda@v2 + # https://github.com/marketplace/actions/setup-micromamba + uses: mamba-org/setup-micromamba@v1 with: - activate-environment: freva-web + environment-name: freva-web environment-file: conda-env.yml - python-version: "3.12" - auto-activate-base: false - - name: Run checks in python - run: make tests - - name: Lint python - run: make lint - - name: Prepare node - run: npm install - - name: Lint js formatting - run: npm run lint-format - - name: Lint javascript - run: npm run lint - - name: Test building the prod system - run: npm run build-production - - name: Test building the dev system - run: npm run build + cache-environment: false + cache-downloads: false + init-shell: bash + - name: Lint js code and python + run: micromamba run -n freva-web make lint + - name: Run build checks for js and python smoke tests + run: micromamba run -n freva-web make tests diff --git a/.gitignore b/.gitignore index 76339d08..f712de07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ *.pyc *.db +*.log virt static_root/img/favicon.svg freva_web.conf +docker/databrowser_api_config.toml db node_modules venv/* @@ -22,3 +24,5 @@ activate_web static/* assets/bundles/* webpack-stats.json +base/migrations/* + diff --git a/.gitmodules b/.gitmodules index a6e09870..69e067c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "docker/config"] path = docker/config url = https://github.com/FREVA-CLINT/freva-service-config.git + branch = main diff --git a/Makefile b/Makefile index 4048377b..c5c1c1f5 100644 --- a/Makefile +++ b/Makefile @@ -2,17 +2,62 @@ export EVALUATION_SYSTEM_CONFIG_FILE := $(PWD)/docker/local-eval-system.conf export EVALUATION_SYSTEM_DRS_CONFIG_FILE := $(PWD)/docker/drs_config.toml -run: - python manage.py runserver +export DJANGO_SUPERUSER_PASSWORD := secret +export DEV_MODE := 1 + +.PHONY: all run runserver runfrontend stopserver stopfrontend stop setup + +all: setup runserver runfrontend + @echo "All services are running in the background." dummy-data: docker/dummy_plugin_runs.sh - python3 docker/solr/ingest_dummy_data.py -lint: + +setup-django: + python manage.py makemigrations base + python manage.py migrate --fake-initial + python manage.py migrate --fake contenttypes + python manage.py createsuperuser --noinput --username admin --email foo@bar.com.au || echo + +setup-node: + npm install + +runserver: + @echo "Starting Django development server..." + python manage.py runserver > runserver.log 2>&1 & + @echo "Django development server is running..." + @echo "To watch the Django server logs, run 'tail -f runserver.log'" + +runfrontend: + @echo "Starting npm development server..." + npm run dev > npm.log 2>&1 & + @echo "npm development server is running..." + @echo "To watch the npm logs, run 'tail -f npm.log'" + +stopserver: + ps aux | grep '[m]anage.py runserver' | awk '{print $$2}' | xargs -r kill + echo "Stopped Django development server..." > runserver.log + +stopfrontend: + pkill -f "npm run dev" + echo "Stopped npm development server..." > npm.log + +stop: stopserver stopfrontend + @echo "All services have been stopped." + +setup: setup-node setup-django dummy-data + +run: runfrontend runserver + +lint: setup-node + npm run lint-format + npm run lint isort -c --profile black -t py312 . -tests: +tests: setup-node + npm run build-production + npm run build rm -rf node_modules pytest -vv $(PWD) tests/ diff --git a/README.md b/README.md index 27c2f0e5..e75ceea4 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,16 @@ conda env create -f conda-env.yml source .envrc ``` -The web ui will need a connection to a solr and mariadb service. -This services can be deployed using +> ``📝`` If conda has issues solving dependencies you can install and use + [mamba](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html) + instead of anaconda. This is recommended because the dependency solvers + that ship with mamba are usually much faster than those conda uses. + +### Additional services running on docker + +The web ui will need a connection to a solr, +[freva-databrowser](https://github.com/FREVA-CLINT/databrowserAPI/) and +mariadb service. This services can be deployed using [`docker-compose`](https://docs.docker.com/compose/install/). ```console @@ -43,7 +51,7 @@ docker-compose down There are some rudimentary tests that check the integration of `django` and the `nodejs` building process. Assuming you have followed steps mentioned above and -created a `freva-dev` cona miniconda environment you can run the tests after +created a `freva-dev` conda miniconda environment you can run the tests after activating this environment: ```console @@ -51,61 +59,105 @@ conda activate freva-web python -m pytest -vv tests ``` -## Django deployment - -You can check if django is working and correctly configured by: - -```console -python manage.py check -``` +## Using GNU `make`: +We have created a Makefile that sets up a development version of the web. You +can use: -If checks are passing issue the following command +- To *setup* / *initialise* the nodejs and django servers use: + ```console + make setup + ``` +- To *run* node and django servers use: + ```console + make run + ``` +- To use both `setup` and `run` command use just use make: + ```console + make + ``` +- To stop the servers use: + ```console + make stop + ``` -```console -python manage.py makemigrations base -python manage.py migrate --fake-initial -python manage.py migrate --fake contenttypes -python manage.py createsuperuser -``` +The django and npm development servers will write output into `runserver.log` and +`npm.log`. You can observe the output of the processes using `tail -f` or something +similar. -The `--fake-inital` flag tells `django` not to create the already existing -database tables. +> ``📝`` You need a Node version of at least 16.5 along a npm version of 8.19 -## Running the server in dev mode +# The Production container +This section only briefly describes the docker image that is automatically +created (see the [next section](#create-a-new-web-release) on how to trigger a +build of the image) and is not meant for actual application. Please install and +use the [freva-deployment package](https://pypi.org/project/freva-deployment/) +to deploy the web app in production mode. -A development server can be set using the following command: +A pre-build image of the web app is available via: ```console -python manage.py runserver [port_number] +docker pull ghcr.io/freva-clint/freva-web:latest ``` -Default port is 8000. If an application is already running on that port you -can change the port number with help of a command line argument - -## Building the JS application : +When running in production mode you should set the following container +environment variables: -**Please note:** You need a Node version of at least 16.5 along a npm version of 8.19 +- ``EVALUATION_SYSTEM_CONFIG_FILE`` : Path to the freva config file +- ``LDAP_USER_DN``: the Distinguished Name within ldap for example + `uid=jdoe,ou=users,dc=example,dc=com`. +- ``LDAP_USER_PW``: password for the LDAP server connection. +- ``DJANGO_SUPERUSER_PASSWORD``: the super user password for the django app. -Install dependencies: +The web app app is running on port 8000, hence you want to publish this port +via the `-p` flag. Ideally the path to the `$EVALUATION_SYSTEM_CONFIG_FILE` +should be mounted into the container as a volume. -```console +Since static files are served by the http web server and not the django web app +you have to add the location of the static files (e.g. `/srv/static`) as a +volume into the container to the `/opt/freva_web/static` location. +On startup of the container django app will create all static files that will +then be available through `/srv/static` on the docker host. -npm install - -``` - -Build project: +All together a minimal working example looks like this: ```console -npm run build -npm run build-production # optimized production build +docker run -it -e EVALUATION_SYSTEM_CONFIG_FILE=/work/freva/evaluation_system.conf \ + -e LDAP_USER_DN='uid=jdoe,ou=users,dc=example,dc=com' \ + -e LDAP_USER_PW='secret' \ + -e DJANGO_SUPERUSER_PASSWORD='more_secret' \ + -v /work/freva:/work/freva:z \ + -v /srv/static:/opt/freva_web/static:z \ + -p 8000:8000 \ + ghcr.io/freva-clint/freva-web:latest ``` +The web app is then available via port 8000 on the host system. -Development: +## Making the web app available on a web server. +To be able to access the web app through a normal http web server you will need +to setup a reverse proxy on your http web server to port 8000. Refer to the +reverse proxy settings for your web server. Here is a minimal example for +apache httpd (using the example from above where the static files are located +in `/srv/static` on the docker host): -```console -npm run dev # starts webpack-dev-server including hot-reloading ``` +Listen 80 +LoadModule proxy_html_module modules/mod_proxy_html.so +LoadModule proxy_module modules/mod_proxy.so +LoadModule proxy_connect_module modules/mod_proxy_connect.so +LoadModule proxy_http_module modules/mod_proxy_http.so + + ProxyPass /static/ ! + ProxyPass /api/databrowser/ http://freva-databrowser.example.com:7777/api/databrowser/ + ProxyPassReverse /api/databrowser/ http://freva-databrowser.example.com:7777/api/databrowser/ + ProxyPass / http://freva.example.com:8000/ + ProxyPassReverse / http://freva.example.com:8000/ + Alias /static /srv/static/ + Alias /robots.txt /srv/static/robots.txt + Alias /favicon.ico /srv/static/favicon.ico + +``` +> ``📝`` This is a minimal example, in a real world scenario you should always + configure your web server to enable web encryption via port 443. # Create a new web release. The production systems are deployed in a docker image hosted on the GitHub diff --git a/assets/js/Components/AccordionItemBody/index.js b/assets/js/Components/AccordionItemBody/index.js index b9691655..f10f4bd1 100644 --- a/assets/js/Components/AccordionItemBody/index.js +++ b/assets/js/Components/AccordionItemBody/index.js @@ -45,7 +45,7 @@ Row.propTypes = { data: PropTypes.array, rowData: PropTypes.shape({ value: PropTypes.string, - count: PropTypes.number, + count: PropTypes.oneOfType([PropTypes.number, PropTypes.string]), }), elemRef: PropTypes.oneOfType([ PropTypes.func, @@ -90,7 +90,9 @@ function AccordionItemBody(props) { let renderedElem; const badge = (
- {count} + + {parseInt(count).toLocaleString("en-US")} +
); const link = ( @@ -165,6 +167,7 @@ function AccordionItemBody(props) { const [windowWidth] = useWindowSize(); const getSize = (index) => sizeMap.current[index] || 24; + const facetName = props.mappedName ?? eventKey; return (
@@ -172,7 +175,7 @@ function AccordionItemBody(props) { className="my-2" id="search" type="text" - placeholder={`Search ${eventKey} name`} + placeholder={`Search ${facetName} name`} onChange={handleChange} />
@@ -212,6 +215,7 @@ function AccordionItemBody(props) { AccordionItemBody.propTypes = { eventKey: PropTypes.string, + mappedName: PropTypes.string, value: PropTypes.array, isFacetCentered: PropTypes.bool, metadata: PropTypes.object, diff --git a/assets/js/Components/Pagination.js b/assets/js/Components/Pagination.js new file mode 100644 index 00000000..af4c4083 --- /dev/null +++ b/assets/js/Components/Pagination.js @@ -0,0 +1,64 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Button } from "react-bootstrap"; + +import { FaChevronLeft, FaChevronRight } from "react-icons/fa"; + +export default function Pagination(props) { + function getPrevious() { + return props.handleSubmit(props.active - 1); + } + + function getNext() { + return props.handleSubmit(props.active + 1); + } + + const firstPage = 1; + const lastPage = props.items; + const activePage = props.active; + const totalFiles = props.totalFiles; + const batchSize = props.batchSize; + + // const valid = active >= 1 && active <= lastPage; + return ( +
+ + + Showing:{" "} + + {((activePage - 1) * batchSize + 1).toLocaleString("en-US")} + {" "} + to + + {" "} + {Math.min(totalFiles, activePage * batchSize).toLocaleString("en-US")} + + {" of "} + {totalFiles}{" "} + + +
+ ); +} + +Pagination.propTypes = { + items: PropTypes.number.isRequired, + active: PropTypes.number.isRequired, + handleSubmit: PropTypes.func.isRequired, + totalFiles: PropTypes.string.isRequired, + batchSize: PropTypes.number.isRequired, +}; diff --git a/assets/js/Containers/App/actions.js b/assets/js/Containers/App/actions.js index 66ceb0eb..3a188b6e 100644 --- a/assets/js/Containers/App/actions.js +++ b/assets/js/Containers/App/actions.js @@ -22,10 +22,10 @@ export const getCurrentUser = () => (dispatch) => { payload: json, }) ) - .catch((error) => { + .catch(() => { dispatch({ type: constants.SET_ERROR, - payload: error, + payload: "Could not load user information.", }); }); }; diff --git a/assets/js/Containers/App/index.js b/assets/js/Containers/App/index.js index d5620502..91c04795 100644 --- a/assets/js/Containers/App/index.js +++ b/assets/js/Containers/App/index.js @@ -2,6 +2,8 @@ import React from "react"; import PropTypes from "prop-types"; import { connect } from "react-redux"; +import { Alert } from "react-bootstrap"; + import Spinner from "../../Components/Spinner"; import { getCurrentUser } from "./actions"; @@ -16,6 +18,13 @@ class App extends React.Component { if (!this.props.currentUser && this.props.error === "") { return ; } + if (this.props.error) { + return ( +
+ {this.props.error} +
+ ); + } return {this.props.children}; } diff --git a/assets/js/Containers/App/reducers.js b/assets/js/Containers/App/reducers.js index f5ceb8d6..ab052c03 100644 --- a/assets/js/Containers/App/reducers.js +++ b/assets/js/Containers/App/reducers.js @@ -12,9 +12,6 @@ export const appReducer = (state = appReducerInitialState, action) => { case constants.SET_ERROR: return { ...state, error: action.payload }; default: - // In our default case we reset the error on - // every new state change if the error has not been - // set. - return { ...state, error: "" }; + return { ...state }; } }; diff --git a/assets/js/Containers/Databrowser/DataBrowserCommand.js b/assets/js/Containers/Databrowser/DataBrowserCommand.js index 782db7b1..a3f09af5 100644 --- a/assets/js/Containers/Databrowser/DataBrowserCommand.js +++ b/assets/js/Containers/Databrowser/DataBrowserCommand.js @@ -7,6 +7,8 @@ import { Button, Card, OverlayTrigger, Tooltip } from "react-bootstrap"; import ClipboardToast from "../../Components/ClipboardToast"; import { copyTextToClipboard } from "../../utils"; +import * as constants from "./constants"; + import { TIME_RANGE_FILE, TIME_RANGE_FLEXIBLE, @@ -38,11 +40,14 @@ function DataBrowserCommandImpl(props) { function getFullCliCommand(dateSelectorToCli) { return ( "freva databrowser " + + (props.selectedFlavour !== constants.DEFAULT_FLAVOUR + ? `--flavour ${props.selectedFlavour} ` + : "") + (props.minDate ? `time=${props.minDate}to${props.maxDate} ` : "") + Object.keys(selectedFacets) .map((key) => { const value = selectedFacets[key]; - return `${key}=${value}`; + return `${props.facetMapping[key]}=${value}`; }) .join(" ") + (dateSelectorToCli && props.minDate @@ -55,6 +60,9 @@ function DataBrowserCommandImpl(props) { return ( 
         freva databrowser
+        {props.selectedFlavour !== constants.DEFAULT_FLAVOUR
+          ? ` --flavour ${props.selectedFlavour} `
+          : ""}
         {props.minDate && (
           
              time=
@@ -68,7 +76,7 @@ function DataBrowserCommandImpl(props) {
           return (
             
               {" "}
-              {key}={value}
+              {props.facetMapping[key]}={value}
             
           );
         })}
@@ -86,10 +94,17 @@ function DataBrowserCommandImpl(props) {
   }
 
   function getFullPythonCommand(dateSelectorToCli) {
-    const args = Object.keys(selectedFacets).map((key) => {
-      const value = selectedFacets[key];
-      return `${key}="${value}"`;
-    });
+    let args = [];
+    if (props.selectedFlavour !== constants.DEFAULT_FLAVOUR) {
+      args.push(`flavour=${props.selectedFlavour}`);
+    }
+    args = [
+      ...args,
+      ...Object.keys(selectedFacets).map((key) => {
+        const value = selectedFacets[key];
+        return `${props.facetMapping[key]}="${value}"`;
+      }),
+    ];
     props.minDate && args.push(`time="${props.minDate} to ${props.maxDate}"`);
     props.minDate && args.push(`time_select="${dateSelectorToCli}"`);
     return `freva.databrowser(${args.join(", ")})`.trimEnd();
@@ -163,6 +178,8 @@ DataBrowserCommandImpl.propTypes = {
   dateSelector: PropTypes.string,
   minDate: PropTypes.string,
   maxDate: PropTypes.string,
+  selectedFlavour: PropTypes.string,
+  facetMapping: PropTypes.object,
 };
 
 const mapStateToProps = (state) => ({
@@ -170,6 +187,8 @@ const mapStateToProps = (state) => ({
   dateSelector: state.databrowserReducer.dateSelector,
   minDate: state.databrowserReducer.minDate,
   maxDate: state.databrowserReducer.maxDate,
+  facetMapping: state.databrowserReducer.facetMapping,
+  selectedFlavour: state.databrowserReducer.selectedFlavour,
 });
 
 export default connect(mapStateToProps)(DataBrowserCommandImpl);
diff --git a/assets/js/Containers/Databrowser/FacetPanel.js b/assets/js/Containers/Databrowser/FacetPanel.js
new file mode 100644
index 00000000..2bd26890
--- /dev/null
+++ b/assets/js/Containers/Databrowser/FacetPanel.js
@@ -0,0 +1,74 @@
+import React from "react";
+import PropTypes from "prop-types";
+
+import { Badge } from "react-bootstrap";
+
+import { initCap, underscoreToBlank } from "../../utils";
+import AccordionItemBody from "../../Components/AccordionItemBody";
+import OwnPanel from "../../Components/OwnPanel";
+
+/*
+This class represents a single Dropdown Menu for a single facet including the title of the facet
+*/
+export function FacetPanel({
+  value,
+  keyVar,
+  metadata,
+  selectedFacets,
+  facetMapping,
+  clickFacet,
+  isFacetCentered,
+}) {
+  const isFacetSelected = !!selectedFacets[keyVar];
+  const facetTitle = initCap(underscoreToBlank(facetMapping[keyVar] ?? keyVar));
+  let panelHeader;
+  if (isFacetSelected) {
+    panelHeader = (
+      
+        {facetTitle}: {selectedFacets[keyVar]}
+      
+    );
+  } else if (value.length === 2) {
+    panelHeader = (
+      
+        {facetTitle}: {value[0]}
+      
+    );
+  } else {
+    const numberOfValues = value.length / 2;
+    panelHeader = (
+      
+        {facetTitle}
+        
+          {numberOfValues.toLocaleString("en-US")}
+        
+      
+    );
+  }
+  return (
+     clickFacet(keyVar) : null}
+    >
+      
+    
+  );
+}
+
+FacetPanel.propTypes = {
+  value: PropTypes.array,
+  keyVar: PropTypes.string,
+  metadata: PropTypes.object,
+  facetMapping: PropTypes.object,
+  selectedFacets: PropTypes.object,
+  isFacetCentered: PropTypes.bool,
+  clickFacet: PropTypes.func.isRequired,
+};
diff --git a/assets/js/Containers/Databrowser/FilesPanel.js b/assets/js/Containers/Databrowser/FilesPanel.js
index d581566f..bfaf0007 100644
--- a/assets/js/Containers/Databrowser/FilesPanel.js
+++ b/assets/js/Containers/Databrowser/FilesPanel.js
@@ -1,14 +1,20 @@
 import React, { useState } from "react";
 import PropTypes from "prop-types";
 import { connect } from "react-redux";
-import { Tooltip, OverlayTrigger, Button, Badge } from "react-bootstrap";
+import { Tooltip, OverlayTrigger, Button } from "react-bootstrap";
+import { withRouter } from "react-router";
 
 import { FaInfoCircle } from "react-icons/fa";
 
+import queryString from "query-string";
+
 import NcdumpDialog, { NcDumpDialogState } from "../../Components/NcdumpDialog";
 import CircularSpinner from "../../Components/Spinner";
 
 import { getCookie } from "../../utils";
+import Pagination from "../../Components/Pagination";
+
+import { BATCH_SIZE } from "./constants";
 
 function FilesPanelImpl(props) {
   const { files, numFiles, fileLoading } = props.databrowser;
@@ -20,6 +26,15 @@ function FilesPanelImpl(props) {
     error: null,
   });
 
+  function setPageOffset(offset) {
+    const currentLocation = props.location.pathname;
+    const query = queryString.stringify({
+      ...props.location.query,
+      start: (offset - 1) * BATCH_SIZE,
+    });
+    props.router.push(currentLocation + "?" + query);
+  }
+
   function loadNcdump(fn, pw) {
     const url = "/api/solr/ncdump/";
     setNcDump({ ...ncdump, status: NcDumpDialogState.LOADING });
@@ -40,7 +55,6 @@ function FilesPanelImpl(props) {
         if (!resp.ok) {
           /* eslint-disable */
           return resp.json().then((json) => {
-            console.log(resp.statusText);
             if (json.error_msg) {
               throw new Error(json.error_msg);
             } else {
@@ -67,12 +81,23 @@ function FilesPanelImpl(props) {
   }
 
   const [filename, setFilename] = useState(null);
+
   return (
     

-      

-        Files
-        {numFiles.toLocaleString("en-US")}
-      

+      
+        

+          Files
+        

+        

+          
+        

+      
       
     ({
   error: state.appReducer.error,
 });
 
-export default connect(mapStateToProps)(FilesPanelImpl);
+export default withRouter(connect(mapStateToProps)(FilesPanelImpl));
diff --git a/assets/js/Containers/Databrowser/FacetDropdown.js b/assets/js/Containers/Databrowser/MetaFacet.js
similarity index 67%
rename from assets/js/Containers/Databrowser/FacetDropdown.js
rename to assets/js/Containers/Databrowser/MetaFacet.js
index 18ccc49f..5529064a 100644
--- a/assets/js/Containers/Databrowser/FacetDropdown.js
+++ b/assets/js/Containers/Databrowser/MetaFacet.js
@@ -7,7 +7,7 @@ import { Badge } from "react-bootstrap";
 import Select from "../../Components/Select";
 import { initCap, underscoreToBlank } from "../../utils";
 
-function FacetDropdownImpl(props) {
+function MetaFacetImpl(props) {
   const options = [];
 
   function constructValueName(category, value) {
@@ -15,9 +15,17 @@ function FacetDropdownImpl(props) {
     const valueInfos = additionalInfo && additionalInfo[value];
     return value + (valueInfos ? " " + valueInfos : "");
   }
+  const primaryFacetsSets = new Set(props.primaryFacets);
+  const sortedFacets = [
+    ...props.primaryFacets,
+    ...Object.keys(props.facetMapping).filter((x) => !primaryFacetsSets.has(x)),
+  ];
 
-  for (const f in props.facets) {
+  for (const f of sortedFacets) {
     const facet = props.facets[f];
+    if (!facet) {
+      continue;
+    }
     for (let i = 0; i < facet.length; i = i + 2) {
       options.push({
         value: constructValueName(f, facet[i]),
@@ -26,10 +34,14 @@ function FacetDropdownImpl(props) {
         label: (
           
    
             
    
-              {initCap(underscoreToBlank(f))}{" "}
+              
+                {initCap(underscoreToBlank(props.facetMapping[f]))}
+              {" "}
               {facet[i]}
             
    
-            {facet[i + 1]}
+            
+              {parseInt(facet[i + 1]).toLocaleString("en-US")}
+            
           
    
         ),
       });
@@ -54,18 +66,22 @@ function FacetDropdownImpl(props) {
   );
 }
 
-FacetDropdownImpl.propTypes = {
+MetaFacetImpl.propTypes = {
   className: PropTypes.string,
   clickFacet: PropTypes.func.isRequired,
   facets: PropTypes.object,
   metadata: PropTypes.object,
   selectedFacets: PropTypes.object,
+  primaryFacets: PropTypes.array,
+  facetMapping: PropTypes.object,
 };
 
 const mapStateToProps = (state) => ({
   facets: state.databrowserReducer.facets,
   metadata: state.databrowserReducer.metadata,
+  primaryFacets: state.databrowserReducer.primaryFacets,
+  facetMapping: state.databrowserReducer.facetMapping,
   selectedFacets: state.databrowserReducer.selectedFacets,
 });
 
-export default connect(mapStateToProps)(FacetDropdownImpl);
+export default connect(mapStateToProps)(MetaFacetImpl);
diff --git a/assets/js/Containers/Databrowser/actions.js b/assets/js/Containers/Databrowser/actions.js
index a3ec8efe..fe507674 100644
--- a/assets/js/Containers/Databrowser/actions.js
+++ b/assets/js/Containers/Databrowser/actions.js
@@ -1,12 +1,11 @@
 import fetch from "isomorphic-fetch";
 
-import queryString from "query-string";
-
 import { getCookie } from "../../utils";
 
 import * as globalStateConstants from "../App/constants";
 
 import * as constants from "./constants";
+import { prepareSearchParams } from "./utils";
 
 export const updateFacetSelection = (queryObject) => (dispatch) => {
   dispatch({
@@ -20,9 +19,33 @@ export const setMetadata = (metadata) => ({
   metadata,
 });
 
-export const loadFacets = (location) => (dispatch) => {
-  dispatch({ type: constants.SET_FACET_LOADING });
-  return fetchResults(dispatch, location, "facet=*", constants.LOAD_FACETS);
+export const setFlavours = () => (dispatch) => {
+  return fetch("/api/databrowser/overview", {
+    credentials: "same-origin",
+    headers: {
+      "X-CSRFToken": getCookie("csrftoken"),
+      Accept: "application/json",
+      "Content-Type": "application/json",
+    },
+  })
+    .then((response) => {
+      if (response.status >= 400) {
+        throw new Error(response.statusText);
+      }
+      return response.json();
+    })
+    .then((json) => {
+      return dispatch({
+        type: constants.SET_FLAVOURS,
+        payload: json,
+      });
+    })
+    .catch(() => {
+      return dispatch({
+        type: globalStateConstants.SET_ERROR,
+        payload: "Internal Error: Could not load flavours",
+      });
+    });
 };
 
 export const loadFiles = (location) => (dispatch) => {
@@ -30,37 +53,14 @@ export const loadFiles = (location) => (dispatch) => {
   return fetchResults(
     dispatch,
     location,
-    "start=0&rows=100",
+    `max-results=${constants.BATCH_SIZE}&translate=false`,
     constants.LOAD_FILES
   );
 };
 
 function fetchResults(dispatch, location, additionalParams, actionType) {
-  // const { selectedFacets, dateSelector, minDate, maxDate } = getState().databrowserReducer;
-  let params = "";
-  if (location) {
-    const queryObject = location.query;
-    const {
-      dateSelector: ignore1,
-      minDate: ignore2,
-      maxDate: ignore3,
-      ...facets
-    } = location.query;
-
-    params = queryString.stringify(facets);
-
-    if (params) {
-      params = "&" + params;
-    }
-
-    const isDateSelected = !!queryObject.minDate;
-    if (isDateSelected) {
-      params += `&time_select=${queryObject.dateSelector}&time=${queryObject.minDate} TO ${queryObject.maxDate}`;
-    }
-  }
-
-  const url = `/solr/solr-search/?${additionalParams}${params}`;
-
+  const searchParams = prepareSearchParams(location, additionalParams);
+  const url = `/api/databrowser/extended_search/${searchParams}`;
   return fetch(url, {
     credentials: "same-origin",
     headers: {
@@ -78,7 +78,11 @@ function fetchResults(dispatch, location, additionalParams, actionType) {
     .then((json) => {
       return dispatch({
         type: actionType,
-        payload: json,
+        payload: {
+          ...json,
+          start: location.query.start ?? 0,
+          flavour: location.query.flavour ?? constants.DEFAULT_FLAVOUR,
+        },
       });
     })
     .catch(() => {
diff --git a/assets/js/Containers/Databrowser/constants.js b/assets/js/Containers/Databrowser/constants.js
index 53db9b2a..c488207b 100644
--- a/assets/js/Containers/Databrowser/constants.js
+++ b/assets/js/Containers/Databrowser/constants.js
@@ -6,7 +6,8 @@ export const UPDATE_FACET_SELECTION = "UPDATE_FACET_SELECTION";
 
 export const SET_METADATA = "SET_METADATA";
 
-export const SET_FACET_LOADING = "SET_FACET_LOADING";
+export const SET_FLAVOURS = "SET_FLAVOURS";
+
 export const SET_FILE_LOADING = "SET_FILE_LOADING";
 
 export const LOAD_NCDUMP = "LOAD_NCDUMP";
@@ -17,3 +18,12 @@ export const RESET_NCDUMP = "RESET_NCDUMP";
 export const TIME_RANGE_FLEXIBLE = "flexible";
 export const TIME_RANGE_STRICT = "strict";
 export const TIME_RANGE_FILE = "file";
+
+export const DEFAULT_FLAVOUR = "freva";
+export const BATCH_SIZE = 100;
+export const ViewTypes = {
+  RESULT_CENTERED: "RESULT_CENTERED",
+  FACET_CENTERED: "FACET_CENTERED",
+};
+
+export const INTAKE_MAXIMUM = 100_000;
diff --git a/assets/js/Containers/Databrowser/index.js b/assets/js/Containers/Databrowser/index.js
index 27070ed5..02787116 100644
--- a/assets/js/Containers/Databrowser/index.js
+++ b/assets/js/Containers/Databrowser/index.js
@@ -7,45 +7,56 @@ import {
   Col,
   Button,
   Alert,
-  Badge,
   OverlayTrigger,
   Tooltip,
+  Form,
+  Collapse,
 } from "react-bootstrap";
 
-import { FaAlignJustify, FaList, FaTimes } from "react-icons/fa";
+import {
+  FaAlignJustify,
+  FaFileExport,
+  FaList,
+  FaMinusSquare,
+  FaPlusSquare,
+  FaTimes,
+} from "react-icons/fa";
 
 import queryString from "query-string";
 import { withRouter } from "react-router";
 
-import AccordionItemBody from "../../Components/AccordionItemBody";
 import OwnPanel from "../../Components/OwnPanel";
 import Spinner from "../../Components/Spinner";
 
 import { initCap, underscoreToBlank } from "../../utils";
 
 import {
-  loadFacets,
   setMetadata,
   loadFiles,
   updateFacetSelection,
+  setFlavours,
 } from "./actions";
 
 import TimeRangeSelector from "./TimeRangeSelector";
 import FilesPanel from "./FilesPanel";
 import DataBrowserCommand from "./DataBrowserCommand";
-import FacetDropdown from "./FacetDropdown";
-
-const ViewTypes = {
-  RESULT_CENTERED: "RESULT_CENTERED",
-  FACET_CENTERED: "FACET_CENTERED",
-};
+import FacetDropdown from "./MetaFacet";
+import { ViewTypes, DEFAULT_FLAVOUR, INTAKE_MAXIMUM } from "./constants";
+import { FacetPanel } from "./FacetPanel";
+import { prepareSearchParams } from "./utils";
 
 class Databrowser extends React.Component {
   constructor(props) {
     super(props);
     this.clickFacet = this.clickFacet.bind(this);
     this.renderFacetBadges = this.renderFacetBadges.bind(this);
-    this.state = { viewPort: ViewTypes.RESULT_CENTERED };
+    const firstViewPort =
+      localStorage.FrevaDatabrowserViewPort ?? ViewTypes.RESULT_CENTERED;
+    localStorage.FrevaDatabrowserViewPort = firstViewPort;
+    this.state = {
+      viewPort: firstViewPort,
+      additionalFacetsVisible: false,
+    };
   }
 
   /**
@@ -53,7 +64,7 @@ class Databrowser extends React.Component {
    * Also load the metadata.js script
    */
   componentDidMount() {
-    this.props.dispatch(loadFacets(this.props.location));
+    this.props.dispatch(setFlavours());
     this.props.dispatch(loadFiles(this.props.location));
     this.props.dispatch(updateFacetSelection(this.props.location.query));
     const script = document.createElement("script");
@@ -80,7 +91,6 @@ class Databrowser extends React.Component {
 
   componentDidUpdate(prevProps) {
     if (prevProps.location.search !== this.props.location.search) {
-      this.props.dispatch(loadFacets(this.props.location));
       this.props.dispatch(loadFiles(this.props.location));
       this.props.dispatch(updateFacetSelection(this.props.location.query));
     }
@@ -94,13 +104,14 @@ class Databrowser extends React.Component {
       // delete
       const { [category]: toRemove, ...queryObject } =
         this.props.location.query;
-      const query = queryString.stringify(queryObject);
+      const query = queryString.stringify({ ...queryObject, start: 0 });
       this.props.router.push(currentLocation + "?" + query);
       return;
     }
     const query = queryString.stringify({
       ...this.props.location.query,
       [category]: value,
+      start: 0,
     });
     if (query) {
       this.props.router.push(currentLocation + "?" + query);
@@ -109,64 +120,69 @@ class Databrowser extends React.Component {
     }
   }
 
-  // dropFacet(category) {
-  //   const currentLocation = this.props.location.pathname;
-  //   const { [category]: toRemove, ...queryObject } = this.props.location.query;
-  //   const query = queryString.stringify(queryObject);
-  //   this.props.router.push(currentLocation + "?" + query);
-  // }
+  clickFlavour(value = DEFAULT_FLAVOUR) {
+    const currentLocation = this.props.location.pathname;
+    const query = queryString.stringify({
+      ...this.props.location.query,
+      flavour: value,
+    });
+    if (query) {
+      this.props.router.push(currentLocation + "?" + query);
+    } else {
+      this.props.router.push(currentLocation);
+    }
+  }
 
   /**
    * Loop all facets and render the panels
    */
   renderFacetPanels() {
-    const { facets, selectedFacets, metadata } = this.props.databrowser;
-    // const { dispatch } = this.props;
+    const { facets, primaryFacets, selectedFacets, facetMapping, metadata } =
+      this.props.databrowser;
 
-    return Object.keys(facets).map((key) => {
+    return primaryFacets.map((key) => {
       const value = facets[key];
-      let panelHeader;
-      const isFacetSelected = !!selectedFacets[key];
-      if (isFacetSelected) {
-        panelHeader = (
-          
-            {initCap(underscoreToBlank(key))}:{" "}
-            {selectedFacets[key]}
-          
-        );
-      } else if (value.length === 2) {
-        panelHeader = (
-          
-            {initCap(underscoreToBlank(key))}: {value[0]}
-          
-        );
-      } else {
-        const numberOfValues = value.length / 2;
-        panelHeader = (
-          
-            {initCap(underscoreToBlank(key))}
-            
-              {numberOfValues}
-            
-          
-        );
-      }
+      if (!value) return undefined;
       return (
-         this.clickFacet(key) : null}
-        >
-          
+      );
+    });
+  }
+
+  renderAdditionalFacets() {
+    const { facets, primaryFacets, selectedFacets, facetMapping, metadata } =
+      this.props.databrowser;
+    const primaryFacetsSet = new Set(primaryFacets);
+
+    return Object.keys(facetMapping)
+      .filter((x) => {
+        return !primaryFacetsSet.has(x);
+      })
+      .map((key) => {
+        const value = facets[key];
+        if (!value) return undefined;
+        return (
+          
-        
-      );
-    });
+        );
+      });
   }
 
   dropTimeSelection() {
@@ -185,24 +201,30 @@ class Databrowser extends React.Component {
     }
   }
 
+  createIntakeLink() {
+    return (
+      "/api/databrowser/intake_catalogue/" +
+      prepareSearchParams(this.props.location, "translate=false")
+    );
+  }
+
   renderTimeSelectionPanel() {
-    const key = "time_range";
     const { dateSelector, minDate, maxDate } = this.props.databrowser;
     const isDateSelected = !!minDate;
     const title = isDateSelected ? (
       
-        Time Range:  
+        Time :  
         
           {dateSelector}: {minDate} to {maxDate}
         
       
     ) : (
-      Time Range
+      Time
     );
     return (
        this.dropTimeSelection() : null}
       >
         
@@ -245,8 +267,8 @@ class Databrowser extends React.Component {
             }}
             key={"selected-" + x + this.props.databrowser.selectedFacets[x]}
           >
-            {initCap(underscoreToBlank(x))}:{" "}
-            {this.props.databrowser.selectedFacets[x]}
+            {initCap(underscoreToBlank(this.props.databrowser.facetMapping[x]))}
+            : {this.props.databrowser.selectedFacets[x]}
             
           
         );
@@ -292,7 +314,9 @@ class Databrowser extends React.Component {
     }
 
     const facetPanels = this.renderFacetPanels();
+    const additionalFacetPanels = this.renderAdditionalFacets();
     const isFacetCentered = this.state.viewPort === ViewTypes.FACET_CENTERED;
+    const flavour = this.props.location.query.flavour;
     return (
       
         
@@ -303,7 +327,52 @@ class Databrowser extends React.Component {
                 
               )}
             
-            
    
+            
    
+               {
+                  this.clickFlavour(x.target.value);
+                }}
+              >
+                {this.props.databrowser.flavours.map((x) => {
+                  return ;
+                })}
+              
+              {this.props.databrowser.numFiles > INTAKE_MAXIMUM ? (
+                
+                      Please narrow down your search to a maximum of 100,000
+                      results in order to enable Intake exports
+                    
+                  }
+                >
+                  
+                    
+                  
+                
+              ) : (
+                
+                  
+                     Intake catalogue
+                  
+                
+              )}
               Change view with facets in focus}
               >
@@ -312,7 +381,13 @@ class Databrowser extends React.Component {
                   variant="outline-secondary"
                   active={isFacetCentered}
                   onClick={() =>
-                    this.setState({ viewPort: ViewTypes.FACET_CENTERED })
+                    this.setState(
+                      { viewPort: ViewTypes.FACET_CENTERED },
+                      () => {
+                        localStorage.FrevaDatabrowserViewPort =
+                          this.state.viewPort;
+                      }
+                    )
                   }
                 >
                   
@@ -325,7 +400,13 @@ class Databrowser extends React.Component {
                   variant="outline-secondary"
                   active={!isFacetCentered}
                   onClick={() =>
-                    this.setState({ viewPort: ViewTypes.RESULT_CENTERED })
+                    this.setState(
+                      { viewPort: ViewTypes.RESULT_CENTERED },
+                      () => {
+                        localStorage.FrevaDatabrowserViewPort =
+                          this.state.viewPort;
+                      }
+                    )
                   }
                 >
                   
@@ -342,6 +423,35 @@ class Databrowser extends React.Component {
 
             {facetPanels}
             {this.renderTimeSelectionPanel()}
+            
+            
+              
    {additionalFacetPanels}
    
+                
           
           
             
@@ -358,10 +468,13 @@ Databrowser.propTypes = {
   databrowser: PropTypes.shape({
     facets: PropTypes.object,
     files: PropTypes.array,
+    flavours: PropTypes.array,
     fileLoading: PropTypes.bool,
     facetLoading: PropTypes.bool,
+    facetMapping: PropTypes.object,
     numFiles: PropTypes.number,
     selectedFacets: PropTypes.object,
+    primaryFacets: PropTypes.array,
     metadata: PropTypes.object,
     dateSelector: PropTypes.string,
     minDate: PropTypes.string,
diff --git a/assets/js/Containers/Databrowser/reducers.js b/assets/js/Containers/Databrowser/reducers.js
index 86eb5081..69b33fde 100644
--- a/assets/js/Containers/Databrowser/reducers.js
+++ b/assets/js/Containers/Databrowser/reducers.js
@@ -4,6 +4,7 @@ const databrowserInitialState = {
   facets: null,
   files: [],
   numFiles: 0,
+  start: 0,
   selectedFacets: {},
   minDate: "",
   maxDate: "",
@@ -11,12 +12,12 @@ const databrowserInitialState = {
   metadata: {},
   facetLoading: false,
   fileLoading: false,
+  flavours: ["freva"],
+  selectedFlavour: constants.DEFAULT_FLAVOUR,
 };
 
 export const databrowserReducer = (state = databrowserInitialState, action) => {
   switch (action.type) {
-    case constants.SET_FACET_LOADING:
-      return { ...state, facetLoading: true };
     case constants.SET_FILE_LOADING:
       return { ...state, fileLoading: true };
     case constants.LOAD_FACETS:
@@ -27,17 +28,8 @@ export const databrowserReducer = (state = databrowserInitialState, action) => {
         facetLoading: false,
       };
     case constants.UPDATE_FACET_SELECTION: {
-      const { minDate, maxDate, dateSelector, ...queryObject } =
+      const { minDate, maxDate, dateSelector, start, flavour, ...queryObject } =
         action.queryObject;
-      // let newObject = {}
-      // if (state.facets) {
-      //   Object.keys(state.facets).forEach(key => {
-      //     if (key in queryObject) {
-      //       newObject = { ...newObject, [key]: queryObject[key] };
-      //     }
-      //   });
-      // }
-      // let dateInfo = {};
       let myMinDate = minDate;
       let myMaxDate = maxDate;
       let myDateSelector = dateSelector;
@@ -49,19 +41,28 @@ export const databrowserReducer = (state = databrowserInitialState, action) => {
       return {
         ...state,
         selectedFacets: { ...queryObject },
+        start: start !== undefined ? parseInt(start) : 0,
         dateSelector: myDateSelector,
         minDate: myMinDate,
         maxDate: myMaxDate,
+        selectedFlavour: flavour || databrowserInitialState.selectedFlavour,
       };
     }
     case constants.SET_METADATA:
       return { ...state, metadata: action.metadata };
+    case constants.SET_FLAVOURS:
+      return { ...state, flavours: action.payload.flavours };
     case constants.LOAD_FILES:
       return {
         ...state,
-        files: action.payload.data,
-        numFiles: action.payload.metadata.numFound,
+        facets: action.payload.facets,
+        primaryFacets: action.payload.primary_facets,
+        facetMapping: action.payload.facet_mapping,
+        files: action.payload.search_results.map((x) => x.file),
+        numFiles: action.payload.total_count,
         fileLoading: false,
+        start: parseInt(action.payload.start),
+        selectedFlavour: action.payload.flavour,
       };
     default:
       return state;
diff --git a/assets/js/Containers/Databrowser/utils.js b/assets/js/Containers/Databrowser/utils.js
new file mode 100644
index 00000000..d26e557a
--- /dev/null
+++ b/assets/js/Containers/Databrowser/utils.js
@@ -0,0 +1,24 @@
+import queryString from "query-string";
+
+import * as constants from "./constants";
+
+export function prepareSearchParams(location, additionalParams = "") {
+  let params = "";
+  let flavourValue = constants.DEFAULT_FLAVOUR;
+  if (location) {
+    const queryObject = location.query;
+    const { dateSelector, minDate, maxDate, flavour, ...facets } = queryObject;
+    flavourValue = flavour ?? flavourValue;
+    params = queryString.stringify(facets);
+
+    if (params && additionalParams) {
+      params = "&" + params;
+    }
+
+    const isDateSelected = !!minDate;
+    if (isDateSelected) {
+      params += `&time_select=${dateSelector}&time=${minDate} TO ${maxDate}`;
+    }
+  }
+  return `${flavourValue}/file?${additionalParams}${params}`;
+}
diff --git a/assets/js/Containers/PluginList/index.js b/assets/js/Containers/PluginList/index.js
index b5445d66..31725d19 100644
--- a/assets/js/Containers/PluginList/index.js
+++ b/assets/js/Containers/PluginList/index.js
@@ -2,6 +2,7 @@ import React from "react";
 import PropTypes from "prop-types";
 import { browserHistory } from "react-router";
 import { connect } from "react-redux";
+import { FaTimes } from "react-icons/fa";
 import {
   Row,
   Col,
@@ -186,7 +187,10 @@ class PluginList extends React.Component {
       children = 
    {error}
    ;
     }
 
-    if (!pluginsLoaded && !this.props.pluginList.errorMessage) {
+    if (
+      (!pluginsLoaded && !this.props.pluginList.errorMessage) ||
+      !currentUser
+    ) {
       return ;
     }
     if (this.props.pluginList.errorMessage) {
@@ -204,7 +208,7 @@ class PluginList extends React.Component {
 
     return (
       
-        
+        
           
             
    Plugins
    
           
@@ -227,14 +231,14 @@ class PluginList extends React.Component {
             
 
         
-          
+          
             {Object.keys(categories).map((key) => {
               return this.renderPluginBlock(filteredPlugins, key);
             })}
           
 
           
-            
+            
               
             
 
-            
    
-              Categories:
+            
    
+              
+                Categories
+                {categoriesFilter.length > 0 && (
+                  
+                    
+                  
+                )}
+              
               
    
                 {Object.keys(categories).map((key) => {
                   return this.renderCategoryCheckbox(
@@ -255,17 +274,40 @@ class PluginList extends React.Component {
                 })}
               
    
             
    
-            
    
-              Tags:
+            
    
+              
+                Tags
+                {tagsFilter.length > 0 && (
+                  
+                    
+                  
+                )}
+              
               
    
                 {tags.map((tag) => {
+                  const variant = _.includes(tagsFilter, tag)
+                    ? "success"
+                    : "secondary";
+                  const disabled = !filteredPlugins.some((x) => {
+                    return _.includes(x[1].tags, tag);
+                  });
                   return (