Remove CQ

README.md

@@ -23,7 +23,7 @@ KAMERA, or the **K**nowledge-guided Image **A**cquisition **M**anag**ER** and **
 ## Layout
 
 - `src/` is the onboard software: ROS packages for the cameras, INS, DAQ and processing, under `src/core`, `src/cams` and `src/process`.
-- `src/cfg/` is the per-system configuration. `nayak/` and `taiga/` each hold a `config.yaml`, the default system state, and one folder per host with its network and clock settings. Files shared by every system sit at the top level. The file `~/kw/SYSTEM_NAME` on a host picks which folder applies.
+- `src/cfg/` is the per-system configuration. `nayak/` and `taiga/` each hold a `config.json`, the default system state, and one folder per host with its network and clock settings. Files shared by every system sit at the top level. The file `~/kw/SYSTEM_NAME` on a host picks which folder applies.
 - `runtime/` is how a host runs the system: the env script, the supervisor program definitions for the leader and the followers, and the scripts supervisor calls (tmux startup). It is the same on every system.
 - `compose/` runs and `docker/` defines the images built for the containers those programs start.
 - `provision/` sets up a host from a fresh install with Ansible, plus the desktop shortcuts and dotfiles it installs.

docker/base/core-deps.dockerfile

@@ -85,11 +85,6 @@ RUN :\
     &&  make -j && make install \
     &&:
 
-# Add yq for configu query to work
-RUN  curl -sL https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 \
-     -o /usr/local/bin/yq && \
-     chmod +x /usr/local/bin/yq
-
 ## ===================  install ebus sdk  ===================
 COPY ./artifacts/ebus.deb /ebus.deb
 COPY ./artifacts/GigE-V-Framework_x86_2.02.0.0132.tar.gz /gigev.tar.gz

docker/base/detector-viame-deps.dockerfile

@@ -37,11 +37,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 # Build tools necessary for catkin and roskv
 
-# Add yq to make config query work
-RUN curl -sL https://github.com/mikefarah/yq/releases/download/2.4.0/yq_linux_amd64 \
-    -o /usr/local/bin/yq && \
-    chmod +x /usr/local/bin/yq
-
 ## === === === === === === === === === === === === === === ===
 # Clone in C++ Deps for Redis
 RUN mkdir /src

docker/core.dockerfile

@@ -15,7 +15,6 @@ RUN rm -rf /entry \
     && printf "\nsource /aliases.sh\n" >> /root/.bashrc
 
 RUN ln -sf $REPO_DIR/scripts/activate_ros.bash $REPO_DIR/activate_ros.bash
-RUN ln -sf $REPO_DIR/src/cfg /cfg
 RUN mkdir -p /root/.config/kamera && \
     ln -sf $REPO_DIR/.dir /root/.config/kamera/repo_dir.bash
 

docker/gui.dockerfile

@@ -29,7 +29,6 @@ RUN rm -rf /entry \
     && printf "\nsource /aliases.sh\n" >> /home/user/.bashrc
 
 RUN ln -sf $REPO_DIR/scripts/activate_ros.bash $REPO_DIR/activate_ros.bash
-RUN ln -sf $REPO_DIR/src/cfg /cfg
 RUN mkdir -p /home/user/.config/kamera && \
     ln -sf $REPO_DIR/.dir /home/user/.config/kamera/repo_dir.bash
 

provision/ansible/playbooks/provision.yml

@@ -37,14 +37,6 @@
                     ]
 
   pre_tasks:
-    - shell: uname -s
-      ignore_errors: yes
-      register: uname_s
-
-    - shell: uname -m
-      ignore_errors: yes
-      register: uname_m
-
     - name: Fully Update System
       become: True
       become_user: root
@@ -174,13 +166,6 @@
         groups: ['user', 'docker', 'dialout',  'sudo']
 
   tasks:
-    - name: Install yq
-      become: True
-      get_url:
-        url: "https://github.com/mikefarah/yq/releases/download/2.4.0/yq_{{ uname_s.stdout }}_amd64"
-        dest: /usr/local/bin/yq
-        mode: '0755'
-
           #    - name: Reboot system
           #      become: True
           #      reboot:

runtime/env.sh

@@ -3,12 +3,12 @@ SYSTEM_NAME="$(cat ${HOME}/kw/SYSTEM_NAME)"
 export SYSTEM_NAME
 
 export KAMERA_DIR=$(${HOME}/.config/kamera/repo_dir.bash)
-CFG_FILE="${KAMERA_DIR}/src/cfg/${SYSTEM_NAME}/config.yaml"
-cq () {
-    CFG_FILE=${CFG_FILE} ${KAMERA_DIR}/src/cfg/get "$@"
-}
+case ":${PATH}:" in
+    *":${KAMERA_DIR}/src/run_scripts/inpath:"*) ;;
+    *) export PATH="${KAMERA_DIR}/src/run_scripts/inpath:${PATH}" ;;
+esac
 
-export REDIS_HOST=$(cq ".redis_host")
+export REDIS_HOST=$(kamera-cfg ".redis_host")
 
 # Uncomment this line if you wish to run the GUI in "offline" mode
 # (without center0, left1, etc. hooked up)
@@ -28,12 +28,12 @@ unset _redis_elapsed
 
 echo "Redis successfully connected at ${REDIS_HOST}, starting."
 
-export ROS_HOSTNAME=$(cq ".master_host")
+export ROS_HOSTNAME=$(kamera-cfg ".master_host")
 export NODE_HOSTNAME=$(hostname)
 export ROS_MASTER_URI="http://${ROS_HOSTNAME}:11311"
 export DOCKER_KAMERA_DIR="/root/kamera"
-export DATA_MOUNT_POINT=$(cq .local_ssd_mnt)
-export CAM_FOV=$(cq ".arch.hosts[\"${NODE_HOSTNAME}\"].fov")
+export DATA_MOUNT_POINT=$(kamera-cfg .local_ssd_mnt)
+export CAM_FOV=$(kamera-cfg ".arch.hosts[\"${NODE_HOSTNAME}\"].fov")
 
 export ROS_DISTRO="noetic"
 export KAMERA_DNS_IP="192.168.88.1"

scripts/docker_images.sh

@@ -12,7 +12,7 @@ KAM_REPO_DIR=$(/home/user/.config/kamera/repo_dir.bash)
 echo $KAM_REPO_DIR
 source "${KAM_REPO_DIR}/runtime/env.sh"
 
-MASTER_HOST=$(cq '.master_host')
+MASTER_HOST=$(kamera-cfg '.master_host')
 
 if [[ -z "$@" ]] ; then
     ARGS=(start)

scripts/kamera_halt.sh

@@ -48,7 +48,7 @@ if [[ -z "${KAM_REPO_DIR}" ]]; then
 fi
 blueprintf "."
 
-MASTER_HOST=$(cq '.master_host')
+MASTER_HOST=$(kamera-cfg '.master_host')
 ## === === === === === ===   End Env setup  === === === === === ===
 
 
@@ -62,8 +62,8 @@ blueprintf "done\nGui down. Killing pods...\n"
 # Bring up all pod systems
 # Query list of hosts as line delim array
 declare -A PIDS
-for host in $(cq '.arch.hosts | keys | join("\n" )') ; do
-    if [[ $(cq ".arch.hosts.${host}.enabled") == 'true' ]]; then
+for host in $(kamera-cfg '.arch.hosts | keys | join("\n" )') ; do
+    if [[ $(kamera-cfg ".arch.hosts.${host}.enabled") == 'true' ]]; then
 	python3 ${KAM_REPO_DIR}/scripts/system.py $host ${ARGS[@]} pod &
         PIDS["${host}_pod"]=$!
 	python3 ${KAM_REPO_DIR}/scripts/system.py $host ${ARGS[@]} detector &

scripts/kamera_run.sh

@@ -60,7 +60,7 @@ blueprintf "Configuring main KAMERA entrypoint."
 source ${KAM_REPO_DIR}/scripts/set_detector_read_state.sh
 blueprintf "."
 
-MASTER_HOST=$(cq '.master_host')
+MASTER_HOST=$(kamera-cfg '.master_host')
 
 for VNAME in MASTER_HOST KAM_REPO_DIR
 do
@@ -81,7 +81,7 @@ until ping -c1 -W1 ${MASTER_HOST} &>/dev/null; do
 done
 
 SKIP_PING=true
-for host in $(cq '.arch.hosts | keys | join("\n" )') ; do
+for host in $(kamera-cfg '.arch.hosts | keys | join("\n" )') ; do
   hostip=$(dig +short $host)
   if [[ $? != 0 ]]; then
     errcho "FATAL: Cannot resolve IP for necassary host ${host}"
@@ -96,9 +96,9 @@ done
 if [[ -n ${SKIP_PING} ]]; then
   echo "all clients located, yay!"
 else
-  for host in $(cq '.arch.hosts | keys | join("\n" )') ; do
+  for host in $(kamera-cfg '.arch.hosts | keys | join("\n" )') ; do
       echo "Waiting on ping $host."
-      if [[ $(cq ".arch.hosts.${host}.enabled") == 'true' ]]; then
+      if [[ $(kamera-cfg ".arch.hosts.${host}.enabled") == 'true' ]]; then
           until ping -c1 -W1 ${host} &>/dev/null; do
               printf "\b${SP:i++%${#SP}:1}"
           done
@@ -119,8 +119,8 @@ else
     echo "NAS mounted!"
 fi
 declare -A PIDS
-for host in $(cq '.arch.hosts | keys | join("\n" )') ; do
-    if [[ $(cq ".arch.hosts.${host}.enabled") == 'true' ]]; then
+for host in $(kamera-cfg '.arch.hosts | keys | join("\n" )') ; do
+    if [[ $(kamera-cfg ".arch.hosts.${host}.enabled") == 'true' ]]; then
 	python3 ${KAM_REPO_DIR}/scripts/system.py $host restart nas &
         PIDS[${host}]=$!
     else
@@ -135,7 +135,7 @@ done
 
 
 # Bring up master and core nodes
-MASTER_HOST=$(cq '.master_host')
+MASTER_HOST=$(kamera-cfg '.master_host')
 blueprintf "done\nBringing up master $MASTER_HOST..."
 python3 ${KAM_REPO_DIR}/scripts/system.py $MASTER_HOST "${ARGS[@]}" master
 
@@ -163,11 +163,11 @@ blueprintf "done\nLaunching pod nodes...\n"
 # Query list of hosts as line delim array
 declare -A PIDS
 # sort, so hosts are started idempotently
-hosts=$(cq '.arch.hosts | keys | join("\n" )')
+hosts=$(kamera-cfg '.arch.hosts | keys | join("\n" )')
 IFS=$'\n' sorted_hosts=($(sort <<<"${hosts[*]}"))
 unset IFS
 for host in "${sorted_hosts[@]}"; do
-    if [[ $(cq ".arch.hosts.${host}.enabled") == 'true' ]]; then
+    if [[ $(kamera-cfg ".arch.hosts.${host}.enabled") == 'true' ]]; then
         # daemon group (kamerad) is always started and never stopped by normal operations
         python3 ${KAM_REPO_DIR}/scripts/system.py $host start daemon
 	python3 ${KAM_REPO_DIR}/scripts/system.py $host "${ARGS[@]}" pod &

src/cfg/README.md

@@ -0,0 +1,57 @@
+# Per-system deployment configs
+
+`<system>/config.json` (taiga, nayak) is the singular source of truth for a
+deployment's static configuration. More static configuration should be
+handled with compose.yml files and .launch files.
+
+It is read in two ways, both of which expect plain JSON:
+
+- shell: `kamera-cfg <query>` (`src/run_scripts/inpath/kamera-cfg`), a
+  plain `jq -r <query> config.json`
+- python: `kamcore/scripts/seed_redis_config.py` and `wxpython_gui.cfg`
+  load it and seed the static keys into redis (config.json always wins for
+  its keys; operator-mutable session state stays owned by the GUI)
+
+These notes used to live as comments inside `config.yaml` and moved here
+when the file became JSON.
+
+## Field notes
+
+**Fields of view.** FOV names describe the position of the camera array's
+reference frame: `{left, right, center, NONE}` (use `NONE` with
+`ROLE=NONE`). Looking along the direction of flight:
+
+```
+ Port     Starbrd
+       _
+      /^\
+    x | | x
+ <==U=| |=U==>
+      | |
+      \ /
+     <<V>>
+ [L]  [C]  [R]   Fields of View
+```
+
+**`.arch`** — camera information and configuration. `max_frame_rate` is the
+max fps we can request of the cameras. `max_mpix: 200000` is 0.2e6.
+
+**`.interfaces`** — maps channels to ethernet interface names (this is
+going to usurp the locations mapping).
+
+**`.locations`** — maps a DEV_ID to a specific FOV. Use this section to
+swap which physical cameras map to a software FOV.
+
+**`.devices`** — configuration for each physical device, keyed by DEV_ID
+(e.g. `ir_n0`). A DEV_ID is an abstract label with no absolute correlation
+to FOV; that indirection makes it easy and less error-prone to reroute
+camera defs. Device blocks appear in groups: cooled FLIR cameras, IR
+"backup" cameras, rgb prosilica backup cameras, rgb phase one cameras, uv
+prosilica cameras. For prosilica GUIDs: the last 3 nybbles of the MAC are
+the GUID, i.e. `02:c2:8c` = 0x02c28c = 180876.
+
+**`.camera_models`** — information about the actual camera devices, keyed
+by model name.
+
+**`.launch`** — expert configs; launch params passed through to the camera
+launch files.