diff --git a/.github/workflows/python_app.yml b/.github/workflows/python_app.yml index 8e5018d..51a5537 100644 --- a/.github/workflows/python_app.yml +++ b/.github/workflows/python_app.yml @@ -20,14 +20,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[all] + pip install .[lse] pip install .[dev] - - name: Lint with flake8 + - name: Lint with ruff run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + ruff check . + ruff format --check . - name: Test with pytest run: | diff --git a/.gitignore b/.gitignore index 3b2cf55..900a33d 100644 --- a/.gitignore +++ b/.gitignore @@ -184,4 +184,8 @@ persist/ data/models/ -.settings.yaml \ No newline at end of file +.settings.yaml + +mlflow.db +.claude +.ruff_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e35b6f5..74d3647 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,20 +14,12 @@ repos: - id: check-yaml - id: debug-statements -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.0 hooks: - - id: flake8 - -- repo: https://github.com/timothycrosley/isort - rev: 5.12.0 - hooks: - - id: isort - -- repo: https://github.com/psf/black - rev: 23.10.1 - hooks: - - id: black + - id: ruff + args: [--fix] + - id: ruff-format # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.9.0 diff --git a/Dockerfile b/Dockerfile index 45a96dd..5ad0f49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,9 @@ WORKDIR /app COPY . /app -RUN pip install --upgrade pip && \ - pip install .[all] +RUN pip install uv && \ + uv pip install --system torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ + uv pip install --system ".[lse]" RUN pip install debugpy diff --git a/README.md b/README.md index 622c55e..f512991 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,59 @@ # Streaming GISAXS Analysis +## Getting Started -## podman tips on ubunut -* Install homebrew -* `brew install podman` -* `brew services start podman` -* `sudo apt install runc` -* use run instead of crun - * vi /etc/containers/containers.conf - * add: +### Local Development with Pixi -``` toml -[system] -runtime = "runc" +This project uses [Pixi](https://pixi.sh) for dependency management and local development. +1. **Install Pixi**: Follow the instructions on the [Pixi website](https://pixi.sh). +2. **Initialize the environment**: + ```bash + pixi install + ``` +3. **Run tests**: + ```bash + pixi run test + ``` + +### Running Locally + +To run the LSE operator locally using the local configuration: + +1. **Start required services**: Ensure you have Redis (or Kvrocks) and Tiled running. You can use the provided `docker-compose.yml` to start them: + ```bash + docker-compose up -d kvrocks tiled + ``` +2. **Run the operator**: + ```bash + pixi run arroyo run block_configs/lse_operator_block_local.yaml + ``` + This uses `block_configs/lse_operator_block_local.yaml` which is configured to connect to services on `localhost`. + +### Beamline Simulator + +The `beamline_sim` service simulates data acquisition by reading logged Tiled events and replaying them. + +To run it locally via Pixi: +```bash +pixi run python -m arroyosas.app.tiled_event_sim_cli tiled_event_logs/run_28b16c3a-5ec4-4c83-8e2e-7e52df303914/ ``` -* install nvidia container [toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) + +When running via Docker Compose, it requires a log directory to be mounted or specified in the container. + +### Deployment + +Changes are typically deployed by updating the Docker images and restarting the services. + +1. **Build the Docker image**: + ```bash + docker build -t arroyosas . + ``` +2. **Deploy using Docker Compose**: + ```bash + docker-compose up -d lse_operator + ``` + The production configuration `block_configs/lse_operator_block.yaml` is used by default in the `docker-compose.yml`. # Copyright diff --git a/block_configs/lse_operator_block.yaml b/block_configs/lse_operator_block.yaml new file mode 100644 index 0000000..4aa6196 --- /dev/null +++ b/block_configs/lse_operator_block.yaml @@ -0,0 +1,24 @@ +blocks: + - name: LSE Operator + description: 2D latent space operator for SAS Data + operator: + class: arroyosas.lse_reduction.operator.build_lse_operator + kwargs: + redis_host: 127.0.0.1 + redis_port: 6666 + + listeners: + - class: arroyosas.tiled.tiled_websocket_bl733.tiled_ws_listener_factory + kwargs: + uri: "https://tiled-staging.computing.als.lbl.gov" + sub_path: "beamlines/bl733/dylan_test" + # api_key: falls back to TILED_LIVE_API_KEY env var + + publishers: + - class: arroyosas.websockets.OneDWSPublisher + kwargs: + host: "0.0.0.0" + port: 8020 + - class: arroyosas.lse_reduction.tiled_results_publisher.tiled_results_publisher_factory + kwargs: + tiled_uri: "https://tiled-staging.computing.als.lbl.gov" diff --git a/block_configs/lse_operator_block_local.yaml b/block_configs/lse_operator_block_local.yaml new file mode 100644 index 0000000..124f549 --- /dev/null +++ b/block_configs/lse_operator_block_local.yaml @@ -0,0 +1,25 @@ +blocks: + - name: LSE Operator + description: 2D latent space operator for SAS Data + operator: + class: arroyosas.lse.build_lse_operator + kwargs: + redis_host: 127.0.0.1 + redis_port: 6666 + + listeners: + - class: arroyosas.tiled.tiled_websocket.tiled_ws_listener_factory + kwargs: + uri: "http://127.0.0.1:8000" + stream_name: "primary" + target: "img" + # api_key: falls back to TILED_LIVE_API_KEY env var + + publishers: + - class: arroyosas.websockets.OneDWSPublisher + kwargs: + host: "0.0.0.0" + port: 8020 + - class: arroyosas.lse_reduction.tiled_results_publisher.tiled_results_publisher_factory + kwargs: + tiled_uri: "http://127.0.0.1:8000" diff --git a/docker-compose.yml b/docker-compose.yml index c7b51bb..3178171 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,14 +4,14 @@ services: tiled: # see the file ./tiled/deploy/config.yml for detailed configuration of tiled - image: ghcr.io/bluesky/tiled:0.1.0-b17 + image: ghcr.io/bluesky/tiled:0.2.8 ports: - - "8000:8000" + - "127.0.0.1:8000:8000" environment: - TILED_SINGLE_USER_API_KEY=${TILED_SINGLE_USER_API_KEY} volumes: - - ./conf/tiled/deploy:/deploy:Z - - ./data:/data:Z + - ./conf/tiled/deploy:/config:z + - ./data:/data:z restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://tiled:8000/healthz"] @@ -22,66 +22,55 @@ services: networks: gisax_net1: beamline_sim: - command: python -m arroyosas.app.tiled_poller_sim_cli + command: python -m arroyosas.app.tiled_event_sim_cli /app/tiled_event_logs image: ghcr.io/als-computing/arroyosas:main restart: unless-stopped volumes: - .:/app:Z - - ./settings.yaml:/app/settings.yaml:Z + - ./settings.yaml:/app/settings.yaml:z + - ./tiled_event_logs/:/app/tiled_event_logs:z networks: gisax_net1: lse_operator: - command: python -m arroyogisaxs.app.kse_operator_cli - build: - context: . - restart: unless-stopped - ports: - - "8021:8021" - volumes: - - .:/app:Z - - ./settings.yaml:/app/settings.yaml:Z - networks: - gisax_net1: - viz_operator: - command: python -m arroyosas.app.viz_operator_cli + command: python -m arroyopy.cli run block_configs/lse_operator_block.yaml build: context: . restart: unless-stopped + environment: + TILED_API_KEY=${TILED_SINGLE_USER_API_KEY} ports: - "8021:8021" volumes: - .:/app:Z - - ./settings.yaml:/app/settings.yaml:Z + - ./block_configs:/app/block_configs:Z networks: gisax_net1: - frontend: - build: - context: . - dockerfile: Dockerfile_frontend - ports: - - "8080:80" - restart: unless-stopped - volumes: - - .:/app:Z - networks: - gisax_net1: + # viz_operator: + # command: python -m arroyosas.app.viz_operator_cli + # build: + # context: . + # restart: unless-stopped + # ports: + # - "8021:8021" + # volumes: + # - .:/app:Z + # - ./settings.yaml:/app/settings.yaml:Z + # networks: + # gisax_net1: - tiled_poller: - command: python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m arroyosas.app.tiled_poller_cli - build: - context: . - dockerfile: Dockerfile - volumes: - - .:/app:Z - - ./settings_container.yaml:/app/settings.yaml:Z - - ./data:/data:Z - ports: - - "5678:5678" - hostname: tiled_poller - networks: - gisax_net1: + # frontend: + # build: + # context: . + # dockerfile: Dockerfile_frontend + # ports: + # - "8080:80" + # restart: unless-stopped + # volumes: + # - .:/app:Z + # networks: + # gisax_net1: kvrocks: diff --git a/frontend/src/components/InputSlider.jsx b/frontend/src/components/InputSlider.jsx index e9372c2..d9956e7 100644 --- a/frontend/src/components/InputSlider.jsx +++ b/frontend/src/components/InputSlider.jsx @@ -110,8 +110,8 @@ export default function InputSlider({
- {/** Optional Input Box on Right of Slider*/} - {showSideInput && + {showSideInput &&
- -

{units}

+

{units}

}
diff --git a/frontend/src/components/Mermaid.jsx b/frontend/src/components/Mermaid.jsx index 7a8cfd1..b8bd520 100644 --- a/frontend/src/components/Mermaid.jsx +++ b/frontend/src/components/Mermaid.jsx @@ -25,14 +25,14 @@ export default function Mermaid() { const graph = ` flowchart LR style Mars1Group stroke:#bbb,stroke-width:4px,stroke-dasharray: 5,5,fill:none; - + btiled(Beamline Tiled) <--http--> framelisten(Frame Listener); - + subgraph Mars1Group["mars1.nsls2.bnl.gov"]; framelisten --zmq frame--> viz_op(Viz Operator); framelisten --zmq frame + tiledurl--> lse_op(LSE Operator); lse_op --websocket--> lse(Latent Space Explorer dash); - + lse <--http--> browser(Browser); viz_op --http 1D reductions--> tiled(Tiled); viz_op --websocket 2 tiled urls + data--> browser; @@ -40,14 +40,14 @@ export default function Mermaid() { lse_op <--zmq raw frame--> lse_worker2(LSE Worker 2); lse_op <--zmq raw frame--> lse_worker3(LSE Worker N); lse_op --http feature vec and url--> tiled; - + browser <--http get data--> tiled; tiled <----> postgres; wf_viz(Workflow Viz) <--http--> browser; wf_viz --> redis(Redis); redis --> viz_op; - - + + end; ` @@ -56,10 +56,10 @@ export default function Mermaid() { const callBackFunction = (event) => { console.log("Clicked element:", event.currentTarget); }; - + // Select all node elements const nodes = document.querySelectorAll("g.node"); - + // Attach the event handler to each node nodes.forEach(node => { node.addEventListener("click", callBackFunction); @@ -111,7 +111,7 @@ export default function Mermaid() { return false; } }; - + const changeBackgroundColor = (element, color="red") => { element.style.fill = color; } @@ -160,7 +160,7 @@ export default function Mermaid() { window.mermaid.contentLoaded(); attachEventHandler(); }); - } + } }, []); @@ -174,7 +174,7 @@ export default function Mermaid() {
- + ) } \ No newline at end of file diff --git a/frontend/src/components/PlotlyHeatMap.jsx b/frontend/src/components/PlotlyHeatMap.jsx index ce820f9..b67afa7 100644 --- a/frontend/src/components/PlotlyHeatMap.jsx +++ b/frontend/src/components/PlotlyHeatMap.jsx @@ -118,9 +118,9 @@ export default function PlotlyHeatMap({ title: yAxisTitle, range: [-0.5, array.length - 0.5], // Dynamically adjust y-axis range autorange: false, - tickmode: showTicks ? 'linear' : '', - tick0: 0, - dtick: showTicks ? tickStep : 10000, + tickmode: showTicks ? 'linear' : '', + tick0: 0, + dtick: showTicks ? tickStep : 10000, showticklabels: showTicks }, autosize: false, @@ -137,8 +137,8 @@ export default function PlotlyHeatMap({ { type: 'line', x0: Math.max(linecutData.x0, 0), - x1: Math.min(linecutData.x1, array[0].length - 1), - y0: linecutData.y0, + x1: Math.min(linecutData.x1, array[0].length - 1), + y0: linecutData.y0, y1: linecutData.y1, line: { color: 'red', diff --git a/frontend/src/components/Status.jsx b/frontend/src/components/Status.jsx index 28ac3a9..b2831ba 100644 --- a/frontend/src/components/Status.jsx +++ b/frontend/src/components/Status.jsx @@ -5,7 +5,7 @@ const images = [ "images/xrayIcon/gif3.png", "images/xrayIcon/gif4.png", ]; - + export default function Status({height='h-14', cycleTime=5, slideshow=true}) { return (
@@ -28,4 +28,3 @@ const images = [
); } - \ No newline at end of file diff --git a/frontend/src/components/TimelineHeatmap.jsx b/frontend/src/components/TimelineHeatmap.jsx index bd69fd8..aed7b14 100644 --- a/frontend/src/components/TimelineHeatmap.jsx +++ b/frontend/src/components/TimelineHeatmap.jsx @@ -29,7 +29,7 @@ export default function TimelineHeatmap ({cumulativeArrayData=[], demo=false}) { ) } - + if (demo) { var sampleCumulativeData = []; const samples = 7; @@ -51,7 +51,6 @@ export default function TimelineHeatmap ({cumulativeArrayData=[], demo=false}) { ) } - + } - \ No newline at end of file diff --git a/frontend/src/components/TimelineHeatmapScatter.jsx b/frontend/src/components/TimelineHeatmapScatter.jsx index 89b3606..4354298 100644 --- a/frontend/src/components/TimelineHeatmapScatter.jsx +++ b/frontend/src/components/TimelineHeatmapScatter.jsx @@ -61,7 +61,7 @@ export default function TimelineHeatmapScatter ({arrayData=[], scatterData=[], d return

Waiting for data

} } - + if (demo) { var sampleCumulativeData = []; const samples = 7; @@ -87,7 +87,6 @@ export default function TimelineHeatmapScatter ({arrayData=[], scatterData=[], d ) } - + } - \ No newline at end of file diff --git a/frontend/src/components/TimelineTiledHeatmapScatter.jsx b/frontend/src/components/TimelineTiledHeatmapScatter.jsx index ff0f316..d726692 100644 --- a/frontend/src/components/TimelineTiledHeatmapScatter.jsx +++ b/frontend/src/components/TimelineTiledHeatmapScatter.jsx @@ -76,12 +76,12 @@ export default function TimelineTiledHeatmapScatter ({tiledLinks=[], demo=false, } if (xValues.length > 0 && yValues.length > 0) { - const newScatterPlot = { + const newScatterPlot = { x: xValues, y: yValues, type: 'scatter', mode: 'lines+markers', - marker: {color: 'red'}, + marker: {color: 'red'}, }; setScatterPlot(newScatterPlot); } @@ -104,9 +104,9 @@ export default function TimelineTiledHeatmapScatter ({tiledLinks=[], demo=false,
{heatmapArray && @@ -128,7 +128,7 @@ export default function TimelineTiledHeatmapScatter ({tiledLinks=[], demo=false, return

Waiting for data

} } - + if (demo) { var sampleCumulativeData = []; const samples = 7; @@ -154,5 +154,5 @@ export default function TimelineTiledHeatmapScatter ({tiledLinks=[], demo=false,
) } - + } diff --git a/frontend/src/hooks/useGISAXS.js b/frontend/src/hooks/useGISAXS.js index a17c8c6..d94f569 100644 --- a/frontend/src/hooks/useGISAXS.js +++ b/frontend/src/hooks/useGISAXS.js @@ -100,7 +100,7 @@ export const useGISAXS = ({}) => { console.log('Exceeded allowable message prints, suppressing future websocket messages'); websocketMessageCount.current = websocketMessageCount.current + 1; } - }/* + }/* var websocketMessage = {} for (var key in newMessage) { if (key === 'raw_frame') { @@ -252,7 +252,7 @@ export const useGISAXS = ({}) => { if (newMessage.msg_type === 'start') { resetAllData(); setIsExperimentRunning(true); - + } if (newMessage.msg_type === 'stop') { setIsExperimentRunning(false); diff --git a/frontend/src/hooks/useGISAXSAlternate.js b/frontend/src/hooks/useGISAXSAlternate.js index 5b464bf..f3cb8ae 100644 --- a/frontend/src/hooks/useGISAXSAlternate.js +++ b/frontend/src/hooks/useGISAXSAlternate.js @@ -154,7 +154,7 @@ export const useGISAXSAlternate = ({}) => { if (newMessage.msg_type === 'start') { resetAllData(); setIsExperimentRunning(true); - + } if (newMessage.msg_type === 'stop') { setIsExperimentRunning(false); diff --git a/frontend/src/pages/Home.jsx b/frontend/src/pages/Home.jsx index d1b78c7..62d81d6 100644 --- a/frontend/src/pages/Home.jsx +++ b/frontend/src/pages/Home.jsx @@ -86,13 +86,13 @@ export default function Home() {
  • {warningMessage.length > 0 ?

    {warningMessage}

    : ''} - {socketStatus === 'closed' - ? -