From 0d655bff3ce4fe2f95765cbfed749c7fae957d5e Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 22 Oct 2026 12:38:09 -0400 Subject: [PATCH] READY: START builds the week's Ableton sets (Sunday and special services) from Planning Center Hold START on Monday: last weekend's songs go to Finished Songs (a changed one parks the version it came from as z...), then each service PCO lists this week (weekend + Special Services / Morning Worship...; First / Together Tuesday in Projects/FIRST TUESDAY) gets 'YYMMDD NA Project' built from the newest weekend set: locators, tempo map and TEMPO clips, Waves Tune cues, songs from Finished Songs (transposed Complex Pro when only another key exists), markers / guide / lighting cues on the dedicated tracks, pads, routing and send A at 0 dB, audio copied in with an Ableton Project Info folder. Co-Authored-By: Claude Opus 5.5 --- README.md | 45 +- ableton_start.py | 1352 ++++++++++++++++++++++++++++++++++++ config.json | 22 + readiness.py | 205 +++++- ready-page.companionconfig | Bin 4905 -> 5040 bytes schema.py | 22 + settings/ui/app.js | 1 + status_panel.py | 5 +- tests/test_readiness.py | 5 +- tests/test_start.py | 377 ++++++++++ 10 files changed, 2027 insertions(+), 7 deletions(-) create mode 100644 ableton_start.py create mode 100644 tests/test_start.py diff --git a/README.md b/README.md index e9e2a73..5d8bcdf 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ HyperDeck is recording). **STATUS PAGE** (bottom left) goes back. | row 0 (Ableton) | TRACKS | FOH MIX | BOH MIX (tap) | LIGHTS | MIDI | SONG KEYS | W TUNE | | row 1 (ProP / Resolume) | PROP | TIMECODE | BKGDS | LED CROSS | LIGHT TIMING | every blinder flash and FX cue (notes 30-39 on `LIGHTING CUES`: Blinder Flash 1-3, Blinder FX 1-5, named from `Ableton/LIGHTING CUES MIDI`) lands on the hit in the `Drums (Live)` stem under it, even when that clip is deactivated because the drummer plays live. Each cue's moment in the stem comes from the tempo map; the Tracks Mac listens in 2.5 ms steps out to almost a 16th either way, finds the strongest hit and where it starts (halfway up its rise); what is under the cue is looked at from 10 ms before it to 25 ms after (the stems' hits start a little after the grid). Each song's cues that sit on solid hits set what "on time" is for that song (its stem's hits start 7-12 ms after the grid). **14 CUES OFF** (amber): a strong hit more than 25 ms from the cue while the cue sits on a quiet spot; a cue already on a solid hit (a double hit follows) is never named. The details say `JEHOVAH bar 405.3.2: Blinder Flash 2 is 145 ms after the hit (about a 16th)` (bars as Live shows them, 4/4). Listened to once per saved set. | moves every cue that is off onto its hit in the open set (all flash/FX notes on that 16th; onto the 16th grid when the hit is within 10 ms of it), keeping where each cue ends, then saves the set (File › Save Live Set in Live, clicked through System Events: `/usr/bin/osascript` has Accessibility on the Tracks Mac since 2026-10-22; `ready.light_autosave`), but only when the set open in Live is this weekend's. If it cannot save, the key says **SAVE SET (n MOVED)** until the set is saved; Cmd+Z in Live takes a move back. Never runs by itself. Uses the status panel's command in AbletonOSC (`tools/abletonosc`: `statuspanel.py` plus a hook at the end of AbletonOSC's `abletonosc/__init__.py`; an AbletonOSC update removes it, so run `tools/abletonosc/install.sh` again, then restart Companion's `ableton` connection) | | row 2 (backstage) | LINEUP TV | ROUTING | IN EARS | | | | | -| row 3 (outgoing) | TRACKS OUT | PROP OUT | a `.probundle` is in this weekend's folder of the Chronicles share (`/Volumes/Location Assets/261025`, read on the ProPresenter Mac). **SHARE NOT MOUNTED**: the share itself is not mounted there (2026-10-22: it had dropped off and the key said NOT IN 261025 although the bundle was there). The ProP Mac keeps it mounted by itself: a LaunchAgent (`org.waterschurch.keep-mounted.location-assets`, from `remote-access/mac-keep-mounted.sh`) looks every minute and at login, mounts it again with the Chronicles login saved in the Keychain, and unmounts and remounts it when it hangs after sleep or a network drop (log: `~/Library/Logs/keep-mounted-location-assets.log` on the ProP Mac). | starts that agent now and waits for the share (about 6 s), then looks again. Also an auto-fix. | +| row 3 (outgoing) | TRACKS OUT | PROP OUT | START | | | | | The READY key at row 3 col 7 (and the one on the status page) shows how many keys still have something to do and the first of them. @@ -163,7 +163,8 @@ everything on the page at once. | LED CROSS | every Resolume column Sunday's playlist calls up has content: each slide's macro (read from the ProPresenter Mac's `Macros` and `.pro` files) sends RossTalk `CC page/row/col` to Companion, that button (and the buttons it presses, e.g. page 17 → page 6 "Arena") sends `/composition/columns/N/connect`; **COL 11 EMPTY** names the column and the items that use it. Also: Resolume on the **Service** deck. | selects the Service deck, then looks again (an empty column is filled in Resolume) | | LINEUP TV | ProdDash's Band Lineup shows this plan, nobody who is not on it or who declined, and everyone on the Band and Communication teams | re-syncs the board from Planning Center | | TRACKS OUT | every location in `"locations"` (AB, SC) has this weekend's project folder | copies NA's project to the locations that have none (as `261018 AB Project`, without `Backup/`). It is copied under a temporary name and renamed at the end, so a half-finished copy never looks sent. It never overwrites a folder that is already there and never runs by itself. | -| PROP OUT | a `.probundle` was saved to `Location Assets/ProPresenter` since Monday | looks again | +| PROP OUT | a `.probundle` is in this weekend's folder of the Chronicles share (`/Volumes/Location Assets/261025`, read on the ProPresenter Mac). **SHARE NOT MOUNTED**: the share itself is not mounted there (2026-10-22: it had dropped off and the key said NOT IN 261025 although the bundle was there). The ProP Mac keeps it mounted by itself: a LaunchAgent (`org.waterschurch.keep-mounted.location-assets`, from `remote-access/mac-keep-mounted.sh`) looks every minute and at login, mounts it again with the Chronicles login saved in the Keychain, and unmounts and remounts it when it hangs after sleep or a network drop (log: `~/Library/Logs/keep-mounted-location-assets.log` on the ProP Mac). | starts that agent now and waits for the share (about 6 s), then looks again. Also an auto-fix. | +| START | every service Planning Center lists this week (this location's Sunday and every special service: First / Together Tuesday, Morning Worship, ...) has its Ableton set. **NO SET SUN 11/1** (red when due): hold START. While it builds the key shows what it is doing (**SAVING JEHOVAH**, **COPYING 40/120**). After a build, for a day: **CONSOLIDATE THE LORD IS** (a song transposed to its PCO key: consolidate it in Live), **NEW SONG GRACE** (not in Finished Songs: build it by hand, page 3 of the workflow PDF). See *START* below. | builds the missing sets (on the Tracks Mac, in the background). Never runs by itself. | **Where the answers come from:** - **Planning Center:** ProdDash's own login on this Mac (`~/Library/Application Support/ProdDash/modules.json`, module `pco-plan`). The plan is today's, else the next one, which is the same plan the lineup TV picks. @@ -175,9 +176,47 @@ everything on the page at once. listens for its answers on NA Ingest, so straight after each read NA Ingest asks it for the tempo and it answers Companion again. -Nothing on the READY page writes to Companion, ProPresenter or the Ableton set file. ROUTING and IN EARS' fix changes +Nothing on the READY page writes to Companion, ProPresenter or an Ableton set file (START only writes new sets and Finished Songs folders). ROUTING and IN EARS' fix changes only the open set in Live (outputs and send A), and only tracks whose name is still where it was a moment before. +### START: the week's Ableton sets (Monday) + +Hold **START** (READY page, row 3 col 3) on Monday. `readiness.py` asks Planning Center for every service from today +until the Thursday after Sunday: NA's weekend service type (`pco_service_type`) and the service types next to it in +PCO (`ready.start_types`: Special Services, Special Events, Morning Worship, Forge & Flourish; not youth, kids or +classes, `ready.start_skip_types`). A plan counts when it has songs with a key. On a Monday that is Sunday and, when +there is one, First / Together Tuesday. Each set is named the year-month-day way plus the location: Sunday 1 Nov 2026 +is `Projects/NA/261101 NA Project`, a First Tuesday is `Projects/FIRST TUESDAY/261103 NA First Tuesday Project`, any +other special service `Projects/NA/ NA Project`. + +`ableton_start.py` then runs on the Tracks Mac (detached, at low priority), doing the workflow PDF's steps: +1. **Last weekend's songs to Finished Songs.** Each song group of the newest weekend set goes to + `Finished Songs/TITLE [KEY]` (key from its locator), with its arrangement markers, lighting cues and guide inside + the group and its audio copied in. A song that matches a saved version (any key, parked or not) is left alone. + A changed one replaces the version it was made from (`CAN'T HELP BUT PRAISE LM [Eb]` for a group called + `CAN'T HELP BUT PRAISE`), which is parked first as `zCAN'T HELP BUT PRAISE LM [Eb]`. +2. **A new set from the newest weekend set:** its songs, pads, Waves Tune cues and marker, guide, loop-guide and + lighting clips go; the groups, the numbered locators (and their computer keys 1-9), the loop locators and the loop + cues stay. Empty loop cues are dropped. +3. **Locators** `N- TITLE - KEY` from PCO: 1 Call to Worship (first song's key and tempo), the songs, then Salvation + (the song after the Salvation moment, a TAG, else the last song). Fewer songs: Salvation moves up a slot (as + "5- SALVATION" did with 3 songs); more: slots are added like the last one. A song too long for its slot pushes the + loop locators, loop cues and later slots along in whole bars, so a loop never fires inside a song. +4. **Tempo map** and one **TEMPO** label clip per tempo (`128.00 BPM`). The tempo is the one the song's audio was + cut at; a different PCO tempo is named in the log. +5. **Waves Tune** key cue at every locator (minor keys use their relative major). +6. **Each song from Finished Songs** at its locator, in its PCO key if that version exists, else the nearest key + transposed the PDF's way (warp on, Complex Pro, formants 0, transpose) and named on the key, because consolidating + is done in Live. Its markers, guide and lighting cues go onto `SECTIONS`, `+GUIDE` and `LIGHTING CUES`. +7. **Pads**: the Call to Worship pad in the first song's key, an ending pad per song in its key, run past its loop, + Salvation's. +8. **Routing** per `ready.routing` and **send A at 0 dB** on every stem. + +Every audio file is copied into the new project (as Collect All and Save does) with an `Ableton Project Info` folder +(Live finds no audio without it: 2026-10-22). A set that is already there is never touched; a new one is built as +`.building` and renamed when complete. Progress: `~/Library/Caches/status-panel-start.json` on the Tracks Mac. +START is on where this location sends its tracks to others (`locations`), or with `ready.start` set. + **Another location:** copy `"ready_page"`, `"ready"`, the four `ready_*` fixes and the READY tile from this config, then change these in `"ready"`: - `pco_service_type` diff --git a/ableton_start.py b/ableton_start.py new file mode 100644 index 0000000..a4e7bf6 --- /dev/null +++ b/ableton_start.py @@ -0,0 +1,1352 @@ +#!/usr/bin/env python3 +"""START: the week's Ableton sets, built the way the worship workflow (Waters_Worship_Ableton_Workflow.pdf) +does them by hand on Mondays. readiness.py sends this file to the Tracks Mac and runs it there, detached +(stdlib only, /usr/bin/python3 3.9), with the services Planning Center lists for the coming days: Sunday and +every special service (First / Together Tuesday, Morning Worship, ...). + + 1 the previous weekend set's songs go to Finished Songs as "TITLE [KEY]": the song group with its + arrangement markers, lighting cues and guide inside it, the audio copied in. A song that changed parks + the old folder as "zTITLE [KEY]" first; an unchanged one is left as it is. + 2 each new set ("261101 NA Project", "261103 NA First Tuesday Project") starts from the newest weekend + set: its songs, pads, Waves Tune cues and marker, guide and lighting clips go; the dedicated groups, + the numbered locators (with their computer-key mappings), the loop locators and the loop cues stay. + 3 the locators are named "N- TITLE - KEY" from PCO, the tempo map and the TEMPO label clips follow the + songs, and a Waves Tune key cue goes at every locator + 4 every song comes in from Finished Songs at its locator. In another key it is transposed the way the + workflow says (warp on, Complex Pro, formants 0, transpose) and flagged, because consolidating it is + done in Live. Its markers, guide and lighting cues go onto the dedicated tracks. + 5 pads: the Call to Worship pad in the first song's key, an ending pad per song past its loop, Salvation + 6 every stem routed per the routing template, send A (in-ears) at 0 dB + +A song too long for its slot pushes the loop locators, loop cues and later slots along (in whole bars); +more songs than slots add slots; fewer move Salvation up, as the sets have done by hand. + +Every audio file the set uses is copied into its project folder (as Collect All and Save does), so the +project can be sent to the locations as it is. Nothing is overwritten: a set that is there is left alone, +and a new set is built in ".building" and only renamed when it is complete. + +Progress and the result go to ~/Library/Caches/status-panel-start.json (the START key on the READY page). +""" +import copy +import gzip +import json +import math +import os +import re +import shutil +import struct +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +STATUS = os.path.expanduser("~/Library/Caches/status-panel-start.json") +LOCK = os.path.expanduser("~/Library/Caches/status-panel-start.lock") +NOTES = {"C": 0, "D": 2, "E": 4, "F": 5, "G": 7, "A": 9, "B": 11} +FINISHED = re.compile(r"^(.*?)\s*\[\s*([A-G](?:#|b)?m?)\s*\]\s*(?:-\s*CUSTOM)?\s*$", re.I) +NUMBERED = re.compile(r"^\s*(\d+)\s*-") +SALVATION = re.compile(r"salvation", re.I) +AUDIO_EVENTS = "DeviceChain/MainSequencer/Sample/ArrangerAutomation/Events" +MIDI_EVENTS = "DeviceChain/MainSequencer/ClipTimeable/ArrangerAutomation/Events" +INITIAL = -63072000.0 # the time Live gives an envelope's starting value + + +# --- small helpers ---------------------------------------------------------------------------- + +def fmt(v): + if isinstance(v, bool): + return "true" if v else "false" + if isinstance(v, float): + if v == int(v) and abs(v) < 1e15: + return str(int(v)) + return repr(round(v, 10)) + return str(v) + + +def val(e, path, default=None): + x = e.find(path) if path else e + return x.get("Value") if x is not None else default + + +def fval(e, path, default=0.0): + v = val(e, path) + return float(v) if v not in (None, "") else default + + +def setv(e, path, v): + x = e.find(path) if path else e + if x is not None: + x.set("Value", fmt(v)) + + +def norm(text): + t = (text or "").upper().replace("&", " AND ").replace("’", "'") + t = re.sub(r"\((LIVE|ACOUSTIC|FEAT[^)]*|RADIO[^)]*)\)", " ", t) + t = re.sub(r"[^A-Z0-9 ]+", " ", t.replace("'", "")) + return " ".join(t.split()) + + +def key_of(text): + m = re.match(r"^\s*([A-Ga-g])(#|b|♯|♭)?\s*(m|min|minor)?\s*$", text or "") + if not m: + return None + pc = NOTES[m.group(1).upper()] + {"#": 1, "♯": 1, "b": -1, "♭": -1}.get(m.group(2) or "", 0) + return pc % 12, bool(m.group(3)) + + +def same_key(a, b, mode=True): + ka, kb = key_of(a), key_of(b) + return bool(ka and kb) and ka[0] == kb[0] and (not mode or ka[1] == kb[1]) + + +def semitones(have, want): + """The shortest transposition from `have` to `want` (-6..+5).""" + a, b = key_of(have), key_of(want) + if not (a and b): + return None + return (b[0] - a[0] + 6) % 12 - 6 + + +def bar_up(x, bar=4.0): + return math.ceil(x / bar - 1e-9) * bar + + +def stem_name(name): + s = re.sub(r"\[.*?\]", " ", (name or "").upper()) + s = re.sub(r"^\s*\d+\s*-\s*", "", s) + return " ".join(s.split()) + + +def template_for(routing, name): + n = stem_name(name) + if n in routing: + return routing[n] + base = re.sub(r"\s*\d+$", "", n) + fam = {tuple(v) for k, v in routing.items() if re.sub(r"\s*\d+$", "", k) == base} + return list(fam.pop()) if len(fam) == 1 else None + + +class Status: + """The job's progress, written where the READY look reads it.""" + + def __init__(self, data): + self.data = data + self.write() + + def write(self, **kw): + self.data.update(kw, time=time.time()) + tmp = STATUS + ".tmp" + with open(tmp, "w") as f: + json.dump(self.data, f) + os.replace(tmp, STATUS) + + +# --- reading and writing sets ---------------------------------------------------------------------- + +def load(path): + with gzip.open(path) as f: + return ET.parse(f).getroot() + + +def save(root, path): + data = '\n' + ET.tostring(root, encoding="unicode") + tmp = path + ".tmp" + with gzip.open(tmp, "wb") as f: + f.write(data.encode("utf-8")) + os.replace(tmp, path) + + +def tracks(ls): + return list(ls.find("Tracks")) + + +def tname(t): + return val(t, "Name/EffectiveName", "") or "" + + +def set_name(t, name): + setv(t, "Name/EffectiveName", name) + setv(t, "Name/UserName", name) + + +def group_of(t): + return val(t, "TrackGroupId", "-1") + + +def family(ls, top): + """`top` and every track under it, in set order.""" + ids, out = {top.get("Id")}, [] + for t in tracks(ls): + if t is top or group_of(t) in ids: + ids.add(t.get("Id")) + out.append(t) + return out + + +def events(t): + """The arrangement's clip list of a track (None for a group or return).""" + return t.find(AUDIO_EVENTS) if t.tag == "AudioTrack" else t.find(MIDI_EVENTS) if t.tag == "MidiTrack" else None + + +def clips(t): + ev = events(t) + return list(ev) if ev is not None else [] + + +def span(c): + return fval(c, "CurrentStart"), fval(c, "CurrentEnd") + + +def warped(c): + return c.tag != "AudioClip" or val(c, "IsWarped") == "true" + + +def seconds(c): + """An unwarped audio clip's length in the sample (its loop values are seconds).""" + return fval(c, "Loop/LoopEnd") - fval(c, "Loop/LoopStart") + + +def clip_tempo(c): + s, e = span(c) + secs = seconds(c) + return (e - s) / secs * 60.0 if secs > 0 and not warped(c) else None + + +def place(c, start, bpm=None, end=None): + """Move a clip to `start`; an unwarped audio clip is as long as its audio lasts at `bpm`.""" + s, e = span(c) + if end is None: + end = start + seconds(c) * bpm / 60.0 if (not warped(c) and bpm) else start + (e - s) + setv(c, "CurrentStart", float(start)) + setv(c, "CurrentEnd", float(end)) + c.set("Time", fmt(float(start))) + + +def renumber(parent, tag=None): + for i, e in enumerate(x for x in parent if tag is None or x.tag == tag): + e.set("Id", str(i)) + + +def is_pointee(e): + return e.tag in ("AutomationTarget", "Pointee") or e.tag.endswith("ModulationTarget") + + +def max_pointee(root): + return max([int(e.get("Id")) for e in root.iter() if is_pointee(e) and (e.get("Id") or "").isdigit()] or [0]) + + +def remap_pointees(elems, next_id): + """New ids for everything automation can point at in `elems` (tracks from another set), and the + envelopes that point at them; returns the next free id.""" + m = {} + for root in elems: + for e in root.iter(): + if is_pointee(e) and e.get("Id") is not None: + m[e.get("Id")] = str(next_id) + e.set("Id", str(next_id)) + next_id += 1 + for root in elems: + for e in root.iter("PointeeId"): + if e.get("Value") in m: + e.set("Value", m[e.get("Value")]) + return next_id + + +def shift_automation(t, off): + for ev in t.findall("AutomationEnvelopes/Envelopes/AutomationEnvelope/Automation/Events"): + for e in ev: + tm = float(e.get("Time", "0")) + if tm > INITIAL + 1: + e.set("Time", fmt(tm + off)) + + +def fit_slots(t, n_scenes): + """As many session clip slots as the set has scenes.""" + for path, tag in (("DeviceChain/MainSequencer/ClipSlotList", "ClipSlot"), + ("DeviceChain/FreezeSequencer/ClipSlotList", "ClipSlot"), ("Slots", "GroupTrackSlot")): + lst = t.find(path) + if lst is None: + continue + slots = list(lst) + while len(slots) > n_scenes: + lst.remove(slots.pop()) + while slots and len(slots) < n_scenes: + s = copy.deepcopy(slots[-1]) + v = s.find("ClipSlot/Value") + if v is not None: + for x in list(v): + v.remove(x) + lst.append(s) + slots.append(s) + renumber(lst) + + +def fit_sends(t, holders, send_a): + """The set's sends (one per return track), send A at `send_a`.""" + sends = t.find("DeviceChain/Mixer/Sends") + if sends is None: + return + have = list(sends) + while len(have) > len(holders): + sends.remove(have.pop()) + while len(have) < len(holders): + h = copy.deepcopy(holders[len(have)]) + sends.append(h) + have.append(h) + renumber(sends) + if have and send_a is not None: + setv(have[0], "Send/Manual", float(send_a)) + + +def route(t, want, targets): + """Output to a template's (upper, lower) display pair, using the routing target the set has for it.""" + r = t.find("DeviceChain/AudioOutputRouting") + target = targets.get(tuple(want)) if want else None + if r is None or target is None: + return False + setv(r, "Target", target) + setv(r, "UpperDisplayString", want[0]) + setv(r, "LowerDisplayString", want[1]) + return True + + +def audio_info(path): + """(frames, sample rate) of a WAV or AIFF file, or None.""" + try: + with open(path, "rb") as f: + head = f.read(12) + if head[:4] == b"RIFF" and head[8:12] == b"WAVE": + rate = align = None + while True: + ch = f.read(8) + if len(ch) < 8: + return None + cid, size = ch[:4], struct.unpack("I", ch[4:])[0] + if cid == b"COMM": + d = f.read(size) + frames = struct.unpack(">I", d[2:6])[0] + exp = struct.unpack(">H", d[8:10])[0] & 0x7FFF + mant = struct.unpack(">Q", d[10:18])[0] + return frames, int(round(mant * 2.0 ** (exp - 16383 - 63))) + f.seek(size + (size % 2), 1) + except (OSError, struct.error, ZeroDivisionError): + return None + return None + + +# --- files: every audio file the set uses goes into its project ------------------------------------------ + +def ref_source(fr, proj): + """The file a FileRef names, wherever it really is now (a moved or renamed project still counts).""" + rel, path = val(fr, "RelativePath") or "", val(fr, "Path") or "" + found = [] + if rel and proj: + found.append(os.path.join(proj, rel)) + if path: + found.append(path if os.path.isabs(path) else os.path.join(proj or "", path)) + if proj and os.path.isabs(path) and "/Samples/" in path: # the project was renamed after saving + found.append(os.path.join(proj, "Samples", path.split("/Samples/", 1)[1])) + return next((p for p in found if os.path.isfile(p)), None) + + +class Files: + """The copies a set needs: (source, project-relative destination), made at the end.""" + + def __init__(self, proj): + self.proj, self.todo, self.missing = proj, {}, [] + + def adopt(self, clip, src_proj, sub): + """Point an audio clip at its own copy under Samples/Imported// in this project.""" + fr = clip.find("SampleRef/FileRef") + if fr is None: + return + src = ref_source(fr, src_proj) + if src is None: + self.missing.append(val(fr, "Path") or val(fr, "RelativePath") or "?") + return + rel = os.path.join("Samples", "Imported", sub, os.path.basename(src)) if sub else os.path.join("Samples", "Imported", os.path.basename(src)) + prev = self.todo.get(rel) + if prev and prev != src: # two different files with one name: keep both + stem, ext = os.path.splitext(rel) + i = 2 + while self.todo.get("%s %d%s" % (stem, i, ext)) not in (None, src): + i += 1 + rel = "%s %d%s" % (stem, i, ext) + self.todo[rel] = src + setv(fr, "RelativePathType", 3) + setv(fr, "RelativePath", rel) + setv(fr, "Path", os.path.join(self.proj, rel)) + + def keep_in_place(self, clip, src_proj): + """A clip of the base set: the same project-relative place in the new project.""" + fr = clip.find("SampleRef/FileRef") + if fr is None: + return + src = ref_source(fr, src_proj) + if src is None: # offline in the base set too: left as it was + return + if src_proj and os.path.abspath(src).startswith(os.path.abspath(src_proj) + os.sep): + rel = os.path.relpath(src, src_proj) + self.todo[rel] = src + if val(fr, "RelativePathType") == "3" or val(fr, "RelativePath"): + setv(fr, "RelativePathType", 3) + setv(fr, "RelativePath", rel) + if os.path.isabs(val(fr, "Path") or ""): + setv(fr, "Path", os.path.join(self.proj, rel)) + else: + self.adopt(clip, src_proj, "") + + def size(self): + return sum(os.path.getsize(s) for s in self.todo.values() if os.path.exists(s)) + + def copy(self, into, progress=None): + done = 0 + for rel, src in sorted(self.todo.items()): + dst = os.path.join(into, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + if not (os.path.exists(dst) and os.path.getsize(dst) == os.path.getsize(src)): + # A plain copy, as Live's Collect All and Save makes: an APFS clone of a Google Drive file + # reads fine in a shell but Live could not open one (2026-10-22). + tmp = dst + ".part" + shutil.copyfile(src, tmp) + os.replace(tmp, dst) + if os.path.exists(src + ".asd") and not os.path.exists(dst + ".asd"): + shutil.copyfile(src + ".asd", dst + ".asd") + done += 1 + if progress: + progress(done, len(self.todo)) + + +# --- the library on the shared drive ----------------------------------------------------------------- + +def finished_index(folders): + """[(folder path, title, key)] of every finished song (not z-parked, not unfinished).""" + out = [] + for d in folders: + d = os.path.expanduser(d) + for n in sorted(os.listdir(d)) if os.path.isdir(d) else []: + m = FINISHED.match(n) + if not m or n.lower().startswith("z") or "UNFINISHED" in n.upper(): + continue + out.append((os.path.join(d, n), m.group(1).strip(), m.group(2))) + return out + + +def finished_als(folder): + found = [os.path.join(folder, n) for n in os.listdir(folder) if n.endswith(".als") and not n.startswith(".")] + return max(found, key=os.path.getmtime) if found else None + + +def close_titles(a, b): + """"CAN'T HELP BUT PRAISE" for "Can't Help But Praise LM", but not "PRAISE" for "Owe You Praise".""" + wa, wb = norm(a).split(), norm(b).split() + short_, long_ = sorted((wa, wb), key=len) + if len(short_) < 2 or not long_: + return False + joined = " " + " ".join(long_) + " " + return (" " + " ".join(short_) + " ") in joined and len(short_) >= 0.6 * len(long_) + + +def pick_version(index, title, key): + """The finished song for a PCO title: in its key if there is one, else the nearest key (to transpose).""" + exact = [x for x in index if norm(x[1]) == norm(title)] + near = [x for x in index if close_titles(x[1], title)] + for pool in (exact, near): + if not pool: + continue + same = [x for x in pool if same_key(x[2], key)] + if same: + return same[0], 0 + scored = sorted(((abs(semitones(x[2], key)) if semitones(x[2], key) is not None else 99, x) for x in pool), + key=lambda sx: sx[0]) + best = scored[0][1] + return best, semitones(best[2], key) or 0 + return None, 0 + + +def pads_index(folder): + folder = os.path.expanduser(folder or "") + out = {} + for n in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + m = re.match(r"^\s*([A-G](?:#|b)?)\s*-", n) + if m and n.lower().endswith((".wav", ".aif", ".aiff")): + out[key_of(m.group(1))[0]] = os.path.join(folder, n) + return out + + +def tune_index(folder): + """{pitch class: (clip name, program)} from the Waves Tune clip library (major keys).""" + folder = os.path.expanduser(folder or "") + out = {} + for n in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + m = re.match(r"^\s*([A-G](?:#|b)?)(?:[:/][A-G](?:#|b)?)?\s*maj\b", n) + if not (m and n.endswith(".alc")): + continue + try: + pg = load(os.path.join(folder, n)).find(".//MidiClip/ProgramChange") + except (OSError, ET.ParseError, EOFError): + continue + if pg is not None: + out[key_of(m.group(1))[0]] = (n[:-4].replace(":", "/"), int(pg.get("Value"))) + return out + + +def tune_for(tunes, key): + """Waves Tune corrects to a major scale: a minor song uses its relative major.""" + k = key_of(key) + if not k: + return None + return tunes.get((k[0] + 3) % 12 if k[1] else k[0]) + + +# --- 1: the previous set's songs to Finished Songs ------------------------------------------------------ + +def song_groups(ls, keep): + """Top-level groups that hold a song (audio clips), not the set's own groups.""" + keep = {norm(k) for k in keep} + out = [] + for t in tracks(ls): + if t.tag != "GroupTrack" or group_of(t) != "-1" or norm(tname(t)) in keep: + continue + fam = family(ls, t) + if any(clips(x) for x in fam if x.tag == "AudioTrack"): + out.append(t) + return out + + +def song_span(fam): + cs = [span(c) for x in fam if x.tag == "AudioTrack" for c in clips(x)] + return (min(s for s, _ in cs), max(e for _, e in cs)) if cs else (None, None) + + +def locator_key(ls, start): + """The key the set's locator gives the song starting at `start` ("3- JEHOVAH - B").""" + for l in ls.findall("Locators/Locators/Locator"): + if abs(fval(l, "Time") - start) <= 8 and NUMBERED.match(val(l, "Name") or ""): + m = re.search(r"-\s*([A-G](?:#|b)?m?)\s*$", val(l, "Name") or "") + if m: + return m.group(1) + return None + + +def tempo_at(ls, beat): + tempo = ls.find("MasterTrack/DeviceChain/Mixer/Tempo") + tid = tempo.find("AutomationTarget").get("Id") + bpm = fval(tempo, "Manual", 120.0) + for env in ls.findall("MasterTrack/AutomationEnvelopes/Envelopes/AutomationEnvelope"): + if val(env, "EnvelopeTarget/PointeeId") == tid: + for e in env.findall("Automation/Events/FloatEvent"): + if float(e.get("Time")) <= beat + 1e-6: + bpm = float(e.get("Value")) + return bpm + + +def stems_signature(root, proj, skip_guide=True): + """What a song is: its stems' file names and sizes (a re-consolidated song has new ones).""" + sig = [] + for c in root.iter("AudioClip"): + fr = c.find("SampleRef/FileRef") + if fr is None: + continue + name = os.path.basename(val(fr, "Path") or val(fr, "RelativePath") or "") + if skip_guide and stem_name(os.path.splitext(name)[0]).startswith("GUIDE"): + continue + stem, ext = os.path.splitext(name) + sig.append((re.sub(r"-\d+$", "", stem) + ext, val(fr, "OriginalFileSize"))) # Collect's "Drums (Live)-1.wav" + return sorted(sig) + + +def helper_copy(src, gid, new_id, keep_from, keep_to, name): + """A dedicated track (SECTIONS, LIGHTING CUES, +GUIDE) cut down to one song, inside its group.""" + t = copy.deepcopy(src) + t.set("Id", str(new_id)) + setv(t, "TrackGroupId", gid) + set_name(t, name) + ev = events(t) + for c in list(ev) if ev is not None else []: + s, e = span(c) + if not (s < keep_to and e > keep_from): + ev.remove(c) + renumber(ev) if ev is not None else None + return t + + +def export_songs(prev_path, cfg, status, log): + """Step 1: every song of the previous weekend set to Finished Songs (new or changed ones only).""" + root = load(prev_path) + ls = root.find("LiveSet") + proj = os.path.dirname(prev_path) + dest_dir = os.path.expanduser(cfg.get("finished_out") or cfg["finished_folders"][0]) + by_name = {norm(tname(t)): t for t in tracks(ls)} + dedicated = {k: by_name.get(norm(cfg[k])) for k in ("sections_track", "lighting_track", "guide_track")} + index = {n.upper(): n for n in os.listdir(dest_dir)} if os.path.isdir(dest_dir) else {} + done = [] + for g in song_groups(ls, cfg["keep_groups"]): + fam = family(ls, g) + start, end = song_span(fam) + title = tname(g).strip() + key = locator_key(ls, start) + if not key or not title: + log.append("NOT SAVED %s: no key on its locator" % title) + continue + mine = stems_signature(_wrap(fam), proj) + versions = list(saved_versions(cfg, dest_dir, title)) + # Saved already, under this name or a near one ("CAN'T HELP BUT PRAISE LM [Eb]" for a group called + # "CAN'T HELP BUT PRAISE"), parked or not: then it came from there and is left as it is. + if any(sig == mine for _, sig in versions): + continue + # Changed: it replaces the version it was made from (most of the same audio, same key), else a new name. + def overlap(sig): + a, b = set(sig), set(mine) + return len(a & b) / float(len(a | b) or 1) + live = [(n, sig) for n, sig in versions if not n.lower().startswith("z") and FINISHED.match(n) and same_key(FINISHED.match(n).group(2), key) + and os.path.dirname(n) == "" and n in index.values()] + best = max(live, key=lambda ns: overlap(ns[1]), default=None) + folder = best[0] if best and overlap(best[1]) >= 0.5 else "%s [%s]" % (title, key) + title = FINISHED.match(folder).group(1).strip() + existing = index.get(folder.upper()) + status.write(step="SAVING %s" % title.upper()[:16]) + out_root = copy.deepcopy(root) + out_ls = out_root.find("LiveSet") + keep_ids = {t.get("Id") for t in fam} + tr = out_ls.find("Tracks") + for t in list(tr): + if t.tag != "ReturnTrack" and t.get("Id") not in keep_ids: + tr.remove(t) + next_tid = max(int(t.get("Id")) for t in tracks(ls)) + 1 + helpers = [] + for k, label in (("sections_track", "SECTIONS"), ("lighting_track", "LIGHTING CUES"), ("guide_track", "Guide")): + if dedicated[k] is not None: + helpers.append(helper_copy(dedicated[k], g.get("Id"), next_tid, start - 0.01, end + 0.01, label)) + next_tid += 1 + first_child = [i for i, t in enumerate(list(tr)) if t.get("Id") in keep_ids][0] + 1 + for i, h in enumerate(helpers): + tr.insert(first_child + i, h) + nxt = remap_pointees(helpers, max_pointee(out_root) + 1) + setv(out_ls, "NextPointeeId", nxt) + for l in out_ls.findall("Locators/Locators/Locator"): + out_ls.find("Locators/Locators").remove(l) + set_tempo(out_ls, [(0.0, tempo_at(ls, start))]) + dest = os.path.join(dest_dir, folder) + building = dest + ".saving" + files = Files(building) + for t in tracks(out_ls): + for c in clips(t) if t.tag == "AudioTrack" else []: + files.adopt(c, proj, "") + shutil.rmtree(building, ignore_errors=True) + make_project(building) + if not cfg.get("no_copy"): + files.copy(building) + # Paths are written for the final folder name. + for c in out_root.iter("AudioClip"): + fr = c.find("SampleRef/FileRef") + if fr is not None and (val(fr, "Path") or "").startswith(building + os.sep): + setv(fr, "Path", dest + val(fr, "Path")[len(building):]) + save(out_root, os.path.join(building, "%s.als" % title)) + if existing: + parked = "z" + existing + while os.path.exists(os.path.join(dest_dir, parked)): + parked = "z" + parked + os.rename(os.path.join(dest_dir, existing), os.path.join(dest_dir, parked)) + log.append("PARKED %s AS %s" % (existing, parked)) + os.rename(building, dest) + index[folder.upper()] = folder + done.append(folder) + if files.missing: + log.append("%s: %d FILES MISSING" % (folder, len(files.missing))) + return done + + +def saved_versions(cfg, dest_dir, title): + """The stems of every saved version of a song (any key, z-parked ones too).""" + dirs = [dest_dir] + [os.path.expanduser(d) for d in cfg["finished_folders"] if os.path.expanduser(d) != dest_dir] + for d in dirs: + for n in sorted(os.listdir(d)) if os.path.isdir(d) else []: + m = FINISHED.match(n.lstrip("zZvV") if n[:1] in "zZ" else n) + if not m or not (norm(m.group(1)) == norm(title) or close_titles(m.group(1), title)): + continue + als = finished_als(os.path.join(d, n)) if os.path.isdir(os.path.join(d, n)) else None + if not als: + continue + try: + yield (n if d == dest_dir else os.path.join(d, n)), stems_signature(load(als), os.path.join(d, n)) + except (OSError, ET.ParseError, EOFError): + continue + + +def make_project(path): + """A Live project folder. Live finds a project's root by its "Ableton Project Info" folder and looks for + the set's audio (relative to the project) from there: without it every clip is offline (2026-10-22).""" + os.makedirs(os.path.join(path, "Ableton Project Info")) + + +def _wrap(elems): + w = ET.Element("Tracks") + w.extend(elems) + return w + + +# --- 2..6: a new set ---------------------------------------------------------------------------- + +def set_tempo(ls, regions): + """The master tempo: a step at the start of each (start, bpm) region.""" + tempo = ls.find("MasterTrack/DeviceChain/Mixer/Tempo") + setv(tempo, "Manual", float(regions[0][1])) + tid = tempo.find("AutomationTarget").get("Id") + env = next((e for e in ls.findall("MasterTrack/AutomationEnvelopes/Envelopes/AutomationEnvelope") + if val(e, "EnvelopeTarget/PointeeId") == tid), None) + if env is None: + return + ev = env.find("Automation/Events") + for e in list(ev): + ev.remove(e) + pts = [(INITIAL, regions[0][1])] + prev = regions[0][1] + for start, bpm in regions[1:]: + if abs(bpm - prev) > 1e-6: + pts += [(start, prev), (start, bpm)] + prev = bpm + for i, (tm, v) in enumerate(pts): + ET.SubElement(ev, "FloatEvent", {"Id": str(i), "Time": fmt(float(tm)), "Value": fmt(float(v))}) + + +class Slot: + def __init__(self, kind, start, loops=None, cues=None, kept=None, locator=None): + self.kind, self.start = kind, start + self.loops = loops or [] # [(rel beat, locator element)] + self.cues = cues or [] # [(rel beat, Loop Cue clip)] + self.kept = kept or [] # [(rel beat, track, clip)] marker/lighting/guide clips that belong to the slot + self.locator = locator + self.song = None + self.length = None # beats the song lasts + self.number = None + self.bpm = None + self.key = None + self.title = None + + +def base_slots(ls, loop_cue, dedicated): + """The base set's slots: each numbered locator, its loop locators, its loop cues and (for Call to + Worship and Salvation) its own marker / lighting / guide clips.""" + locs = sorted(ls.findall("Locators/Locators/Locator"), key=lambda l: fval(l, "Time")) + heads = [l for l in locs if NUMBERED.match(val(l, "Name") or "")] + if not heads: + raise RuntimeError("THE BASE SET HAS NO NUMBERED LOCATORS") + slots = [] + for i, h in enumerate(heads): + start = fval(h, "Time") + end = fval(heads[i + 1], "Time") if i + 1 < len(heads) else 1e12 + kind = "ctw" if i == 0 else "sal" if SALVATION.search(val(h, "Name") or "") else "song" + s = Slot(kind, start, locator=h) + s.loops = [(fval(l, "Time") - start, l) for l in locs + if l not in heads and (val(l, "Name") or "").strip().upper().startswith("LOOP") and start <= fval(l, "Time") < end] + s.cues = [(span(c)[0] - start, c) for c in (clips(loop_cue) if loop_cue is not None else []) + if start <= span(c)[0] < end and c.find(".//MidiNoteEvent") is not None] + if kind != "song": + for t in dedicated: + s.kept += [(span(c)[0] - start, t, c) for c in clips(t) if start <= span(c)[0] < end] + s.base_end = end + slots.append(s) + if slots[-1].kind != "sal": + slots[-1].kind = "sal" + return slots + + +def plan_layout(slots, songs, sal): + """Slots for Call to Worship, the songs and Salvation, pushed along where a song is too long.""" + ctw = slots[0] + base_songs = [s for s in slots if s.kind == "song"] + sal_slot = [s for s in slots if s.kind == "sal"][-1] + out = [ctw] + ctw.slot_len = (base_songs[0].start if base_songs else sal_slot.start) - ctw.start + delta = 0.0 + for i, song in enumerate(songs): + if i < len(base_songs): + s = copy.copy(base_songs[i]) + s.start = base_songs[i].start + delta + gap = (base_songs[i + 1].start if i + 1 < len(base_songs) else sal_slot.start) - base_songs[i].start + else: # more songs than the base set had slots: like the last one + tmpl = base_songs[-1] if base_songs else sal_slot + s = copy.copy(tmpl) + prev = out[-1] + s.start = bar_up(prev.start + prev.slot_len) + first = min([r for r, _ in tmpl.loops] or [0.0]) + s.loops = [(r - first, l) for r, l in tmpl.loops] + s.cues = [(r - first, c) for r, c in tmpl.cues] + gap = None + s.loops, s.cues = list(s.loops), list(s.cues) + s.song, s.length = song, song.get("length") + length = s.length or 0.0 + first_loop = min([r for r, _ in s.loops] + [r for r, _ in s.cues]) if (s.loops or s.cues) else None + if gap is None: # a new slot: its loop right after the song + push = bar_up(length + 4.0) + elif first_loop is not None and length + 4.0 > first_loop: + push = bar_up(length + 4.0 - first_loop) + else: + push = 0.0 + if push: + s.loops = [(r + push, l) for r, l in s.loops] + s.cues = [(r + push, c) for r, c in s.cues] + tail = max([r for r, _ in s.cues] + [r for r, _ in s.loops] + [length]) + s.slot_len = (gap + push) if gap is not None else bar_up(tail + 12.0) + if gap is not None: + delta += push + out.append(s) + # Salvation: in the first slot no song took (as "5- SALVATION" did when there were 3 songs), else after the last. + s = copy.copy(sal_slot) + s.loops, s.cues, s.kept = list(sal_slot.loops), list(sal_slot.cues), list(sal_slot.kept) + if len(songs) < len(base_songs): + s.start = base_songs[len(songs)].start + delta + elif len(songs) == len(base_songs): + s.start = sal_slot.start + delta + else: + prev = out[-1] + s.start = bar_up(prev.start + prev.slot_len) + s.song = sal + s.slot_len = max([r for r, _ in s.cues] + [r for r, _ in s.loops] + [64.0]) + 32.0 + ctw.slot_len = out[1].start - ctw.start if len(out) > 1 else s.start - ctw.start + out.append(s) + for n, slot in enumerate(out, 1): + slot.number = n + return out + + +def clone_clip(tmpl, start, end, name): + c = copy.deepcopy(tmpl) + setv(c, "CurrentStart", float(start)) + setv(c, "CurrentEnd", float(end)) + c.set("Time", fmt(float(start))) + setv(c, "Name", name) + return c + + +def pad_clip(tmpl, path, start, end, bpm): + """An ambient pad, unwarped (as the sets have them), from the start of the file.""" + c = clone_clip(tmpl, start, end, os.path.splitext(os.path.basename(path))[0]) + info = audio_info(path) + total = info[0] / float(info[1]) if info else 600.0 + secs = min((end - start) * 60.0 / bpm, total) + setv(c, "CurrentEnd", float(start + secs * bpm / 60.0)) + setv(c, "IsWarped", False) + for k, v in (("LoopStart", 0.0), ("LoopEnd", secs), ("StartRelative", 0.0), ("LoopOn", False), + ("OutMarker", total), ("HiddenLoopStart", 0.0), ("HiddenLoopEnd", total)): + setv(c, "Loop/" + k, v) + sr = c.find("SampleRef") + fr = sr.find("FileRef") + setv(fr, "Path", path) + setv(fr, "RelativePath", "") + setv(fr, "RelativePathType", 0) + setv(fr, "OriginalFileSize", os.path.getsize(path)) + setv(fr, "OriginalCrc", 0) + setv(sr, "LastModDate", int(os.path.getmtime(path))) + if info: + setv(sr, "DefaultDuration", info[0]) + setv(sr, "DefaultSampleRate", info[1]) + wm = c.find("WarpMarkers") + if wm is not None: + for x in list(wm): + wm.remove(x) + for i, (sec, beat) in enumerate(((0.0, 0.0), (total, total * bpm / 60.0))): + ET.SubElement(wm, "WarpMarker", {"Id": str(i), "SecTime": fmt(sec), "BeatTime": fmt(beat)}) + return c + + +def transpose(c, semis, bpm): + """The workflow's transpose: warp on, Complex Pro, formants 0, transpose (consolidating is left to Live).""" + if not warped(c): + s, e = span(c) + ls_, le = fval(c, "Loop/LoopStart"), fval(c, "Loop/LoopEnd") + total = fval(c, "Loop/HiddenLoopEnd", le) + to_beat = bpm / 60.0 + setv(c, "IsWarped", True) + for k, v in (("LoopStart", ls_ * to_beat), ("LoopEnd", le * to_beat), ("StartRelative", 0.0), + ("OutMarker", total * to_beat), ("HiddenLoopStart", 0.0), ("HiddenLoopEnd", total * to_beat)): + setv(c, "Loop/" + k, v) + wm = c.find("WarpMarkers") + if wm is not None: + for x in list(wm): + wm.remove(x) + for i, (sec, beat) in enumerate(((0.0, 0.0), (max(total, 1.0), max(total, 1.0) * to_beat))): + ET.SubElement(wm, "WarpMarker", {"Id": str(i), "SecTime": fmt(sec), "BeatTime": fmt(beat)}) + setv(c, "WarpMode", 6) + setv(c, "ComplexProFormants", 0) + setv(c, "PitchCoarse", int(semis)) + + +def dated(t): + """The consolidation stamp in a track's name ("4-Guide [2026-09-28 141814]"), for picking the newest.""" + m = re.search(r"\[(\d{4}-\d\d-\d\d \d{6})\]", tname(t)) + return m.group(1) if m else "" + + +def song_parts(fam): + """A finished song's group split into its stems and the three helper tracks the workflow puts in it + (one of each: a song saved twice can hold two guides; the newest is the one).""" + stems, sections, lighting, guide = [], [], [], [] + for t in fam[1:]: + cs = clips(t) + if t.tag == "MidiTrack": + if any((val(c, "Name") or "").upper().startswith("LIGHTING CUES") for c in cs) or "LIGHT" in tname(t).upper(): + lighting.append(t) + else: + sections.append(t) + elif t.tag == "AudioTrack" and stem_name(tname(t)).startswith("GUIDE"): + guide.append(t) + elif t.tag in ("AudioTrack", "GroupTrack"): + stems.append(t) + pick = lambda ts, key: clips(max(ts, key=key)) if ts else [] + return (stems, pick(sections, lambda t: len(clips(t))), pick(lighting, lambda t: len(clips(t))), + pick(guide, lambda t: (dated(t), -fam.index(t)))) + + +def load_song(entry, cfg): + """A finished song: its tracks, helper clips, start beat and the tempo its audio was cut at.""" + folder = entry[0] + als = finished_als(folder) + if not als: + return None + root = load(als) + ls = root.find("LiveSet") + tops = [t for t in tracks(ls) if t.tag == "GroupTrack" and group_of(t) == "-1"] + if not tops: + return None + fam = family(ls, tops[0]) + stems, sections, lighting, guide = song_parts(fam) + audio = [c for t in stems for c in clips(t) if c.tag == "AudioClip"] + if not audio: + return None + start = min(span(c)[0] for c in audio) + tempos = sorted(t for t in (clip_tempo(c) for c in audio) if t) + bpm = tempos[len(tempos) // 2] if tempos else tempo_at(ls, start) + end = max(start + (seconds(c) * bpm / 60.0 if not warped(c) else span(c)[1] - span(c)[0]) for c in audio) + return {"root": root, "ls": ls, "proj": folder, "group": tops[0], + "stems": stems, "sections": sections, "lighting": lighting, "guide": guide, + "start": start, "bpm": round(bpm, 2), "length": end - start, "folder": os.path.basename(folder), + "title": entry[1], "key": entry[2]} + + +def build_set(svc, base_path, out_dir, name, cfg, status, log): + """Steps 2-6 for one service; returns a summary.""" + root = load(base_path) + ls = root.find("LiveSet") + base_proj = os.path.dirname(base_path) + final_proj = os.path.join(out_dir, name) + building = final_proj + ".building" + files = Files(final_proj) + by_name = {norm(tname(t)): t for t in tracks(ls)} + + def track(k): + return by_name.get(norm(cfg[k])) + loop_cue, pads_t, tune_t, tempo_t = track("loop_cue_track"), track("pad_track"), track("tune_track"), track("tempo_track") + dedicated = [t for t in (track("sections_track"), track("lighting_track"), track("guide_track")) if t is not None] + cleared = [t for t in (pads_t, tune_t, track("loopguide_track")) if t is not None] + dedicated + tmpl = {"pad": (clips(pads_t) or [None])[0] if pads_t is not None else None, + "tune": (clips(tune_t) or [None])[0] if tune_t is not None else None, + "tempo": (clips(tempo_t) or [None])[0] if tempo_t is not None else None, + "cue": (clips(loop_cue) or [None])[0] if loop_cue is not None else None} + slots = base_slots(ls, loop_cue, dedicated) + + # The songs, from Finished Songs. + index = finished_index(cfg["finished_folders"] + ([cfg["finished_out"]] if cfg.get("finished_out") else [])) + songs = [] + for s in svc["songs"]: + entry, semis = pick_version(index, s["title"], s["key"]) + got = load_song(entry, cfg) if entry else None + info = {"title": s["title"], "key": s["key"], "pco_bpm": s.get("bpm")} + if got: + info.update(got, semis=semis, key=s["key"], from_key=got["key"]) + if semis: + log.append("TRANSPOSE %s %s>%s: CONSOLIDATE IN LIVE" % (got["title"].upper(), got["key"], s["key"])) + if s.get("bpm") and abs(float(s["bpm"]) - got["bpm"]) > 0.6: + log.append("TEMPO %s: PCO %s, SONG %g" % (got["title"].upper(), s["bpm"], got["bpm"])) + else: + info.update(length=None, bpm=float(s.get("bpm") or 0) or None) + log.append("NEW SONG %s: BUILD IT BY HAND" % s["title"].upper()) + songs.append(info) + if not songs: + raise RuntimeError("NO SONGS IN PCO") + first_bpm = next((x["bpm"] for x in songs if x.get("bpm")), 120.0) + sal = svc.get("salvation") or {} + sal_song = next((x for x in songs if sal.get("title") and norm(x["title"]) == norm(sal["title"])), None) + sal_bpm = (sal_song or {}).get("bpm") or (float(sal["bpm"]) if sal.get("bpm") else None) or songs[-1].get("bpm") or first_bpm + sal_key = sal.get("key") or songs[-1]["key"] + layout = plan_layout(slots, songs, {"title": "SALVATION", "key": sal_key, "bpm": sal_bpm}) + + # Where each display pair ("Ext. Out", "7/8 GTRS") really goes, from every track before any go. + targets = {} + for t in tracks(ls): + r = t.find("DeviceChain/AudioOutputRouting") + if r is not None and val(r, "UpperDisplayString"): + targets.setdefault((val(r, "UpperDisplayString"), val(r, "LowerDisplayString") or ""), val(r, "Target")) + # Out with last week: song groups and the per-song clips. + tr = ls.find("Tracks") + insert_at = None + for g in song_groups(ls, cfg["keep_groups"]): + for t in family(ls, g): + if insert_at is None: + insert_at = list(tr).index(t) + tr.remove(t) + for t in cleared + ([loop_cue] if loop_cue is not None else []) + ([tempo_t] if tempo_t is not None else []): + ev = events(t) + for c in list(ev) if ev is not None else []: + ev.remove(c) + loc_parent = ls.find("Locators/Locators") + numbered = {int(NUMBERED.match(val(l, "Name")).group(1)): l for l in loc_parent if NUMBERED.match(val(l, "Name") or "")} + for l in list(loc_parent): + loc_parent.remove(l) + + n_scenes = len(list(ls.find("Scenes"))) + holders = [] + for t in tracks(ls): + sends = t.find("DeviceChain/Mixer/Sends") + if sends is not None and len(sends): + holders = list(sends) + break + routing = {stem_name(k): v for k, v in (cfg.get("routing") or {}).items()} + next_tid = max(int(t.get("Id")) for t in tracks(ls)) + 1 + next_ptr = max(max_pointee(root), int(val(ls, "NextPointeeId", "0"))) + 1 + if insert_at is None: + rets = [i for i, t in enumerate(list(tr)) if t.tag == "ReturnTrack"] + insert_at = rets[0] if rets else len(list(tr)) + + def add_clip(t, c): + ev = events(t) + ev.append(c) + renumber(ev) + + regions, summary = [], {"name": name, "songs": [], "service": svc.get("label")} + for slot in layout: + s = slot.start + song = slot.song if slot.kind == "song" else None + if slot.kind == "ctw": + slot.key, slot.bpm, slot.title = songs[0]["key"], first_bpm, "CALL TO WORSHIP" + elif slot.kind == "sal": + slot.key, slot.bpm, slot.title = sal_key, sal_bpm, "SALVATION" + else: + slot.key, slot.bpm = song["key"], song.get("bpm") or first_bpm + slot.title = (song.get("title") or song["title"]).upper() + regions.append((s, slot.bpm)) + # Locator: the base set's locator with this number (it carries the computer key), else a new one. + loc = numbered.get(slot.number) + if loc is None: + loc = copy.deepcopy(slot.locator) + pk = loc.find("Mapping/PersistentKeyString") + if pk is not None: + pk.set("Value", str(slot.number) if slot.number < 10 else "") + for k in ("Mapping/NoteOrController", "Mapping/LowerRangeNote", "Mapping/UpperRangeNote"): + setv(loc, k, -1) + setv(loc, "Mapping/IsNote", False) + setv(loc, "Time", float(s)) + setv(loc, "Name", "%d- %s - %s" % (slot.number, slot.title, slot.key)) + loc_parent.append(loc) + for r, l in slot.loops: + l = copy.deepcopy(l) + setv(l, "Time", float(s + r)) + loc_parent.append(l) + if loop_cue is not None: + for r, c in slot.cues: + cc = copy.deepcopy(c) + place(cc, s + r) + add_clip(loop_cue, cc) + for r, t, c in slot.kept: + cc = copy.deepcopy(c) + place(cc, s + r) + if cc.tag == "AudioClip": + files.keep_in_place(cc, base_proj) + add_clip(t, cc) + # Waves Tune key cue at the locator. + tune = tune_for(cfg["_tunes"], slot.key) + if tune_t is not None and tmpl["tune"] is not None: + if tune: + c = clone_clip(tmpl["tune"], s, s + 1, tune[0]) + setv(c, "ProgramChange", tune[1]) + add_clip(tune_t, c) + else: + log.append("NO WAVES TUNE CUE FOR %s" % slot.key) + # The song. + if song and song.get("root") is not None: + off = s - song["start"] + fam_src = [song["group"]] + song["stems"] + new = [copy.deepcopy(t) for t in fam_src] + idmap = {} + for t in new: + idmap[t.get("Id")] = str(next_tid) + t.set("Id", str(next_tid)) + next_tid += 1 + for t in new: + if group_of(t) in idmap: + setv(t, "TrackGroupId", idmap[group_of(t)]) + fit_slots(t, n_scenes) + fit_sends(t, holders, cfg.get("send_a", 1.0)) + shift_automation(t, off) + for c in clips(t): + place(c, span(c)[0] + off, song["bpm"]) + if c.tag == "AudioClip": + if song.get("semis"): + transpose(c, song["semis"], song["bpm"]) + files.adopt(c, song["proj"], song["folder"]) + if t.tag == "AudioTrack": + want = template_for(routing, tname(t)) if routing else None + if want and want[0] != "Master" and not route(t, want, targets): + log.append("NO ROUTE %s FOR %s" % ("/".join(want), stem_name(tname(t)))) + set_name(new[0], song["title"].upper()) + next_ptr = remap_pointees(new, next_ptr) + for i, t in enumerate(new): + tr.insert(insert_at + i, t) + insert_at += len(new) + for kind, k in (("sections", "sections_track"), ("lighting", "lighting_track"), ("guide", "guide_track")): + t = track(k) + if t is None: + continue + for c in song[kind]: + cc = copy.deepcopy(c) + place(cc, span(c)[0] + off, song["bpm"]) + if cc.tag == "AudioClip": + files.adopt(cc, song["proj"], song["folder"]) + a, b = span(cc) + if (cc.tag == "AudioClip") != (t.tag == "AudioTrack"): + continue + if any(span(x)[0] < b - 0.01 and span(x)[1] > a + 0.01 for x in clips(t)): + log.append("%s: A %s CLIP OVERLAPS, LEFT OUT" % (song["title"].upper(), cfg[k].upper())) + continue + add_clip(t, cc) + summary["songs"].append({"title": song["title"], "key": song["key"], "from": song["folder"], + "semis": song.get("semis", 0), "bpm": song["bpm"], "at": s}) + elif song: + summary["songs"].append({"title": song["title"], "key": song["key"], "from": None, "at": s}) + # Pads. + pad = cfg["_pads"].get((key_of(slot.key) or (None,))[0]) + if pads_t is not None and tmpl["pad"] is not None and pad: + last_cue = max([r for r, _ in slot.cues] + [r for r, _ in slot.loops] + [0.0]) + nxt = layout[layout.index(slot) + 1].start if slot is not layout[-1] else None + if slot.kind == "ctw": + a = s + b = min(nxt + 40.0, nxt + (layout[1].length or 40.0)) if nxt is not None else s + last_cue + 24 + elif slot.kind == "sal": + a, b = s, s + last_cue + 24.0 + elif not song.get("length"): # a song still to build: only the loop gets a pad + first = min([r for r, _ in slot.loops] + [r for r, _ in slot.cues] + [last_cue]) + a, b = s + max(first - 32.0, 0.0), s + last_cue + 24.0 + if nxt is not None: + b = min(b, nxt - 4.0) + else: + end = s + (song.get("length") or 0.0) + a = max(s, end - 64.0) + b = s + last_cue + 24.0 + if nxt is not None: + b = min(b, nxt - 4.0) + b = max(b, end + 8.0) + c = pad_clip(tmpl["pad"], pad, a, b, slot.bpm) + files.adopt(c, None, "") + add_clip(pads_t, c) + elif pads_t is not None and not pad: + log.append("NO PAD IN %s" % slot.key) + + # Tempo map and the TEMPO label clips (one per run of the same tempo). + set_tempo(ls, regions) + if tempo_t is not None and tmpl["tempo"] is not None: + last = layout[-1] + end_all = last.start + max([r for r, _ in last.cues] + [r for r, _ in last.loops] + [64.0]) + 32.0 + runs = [] + for start, bpm in regions: + if runs and abs(runs[-1][1] - bpm) < 1e-6: + continue + runs.append([start, bpm]) + for i, (start, bpm) in enumerate(runs): + end = runs[i + 1][0] if i + 1 < len(runs) else end_all + c = clone_clip(tmpl["tempo"], start, end, "%.2f BPM" % bpm) + files.keep_in_place(c, base_proj) + add_clip(tempo_t, c) + # Clips the base set keeps (LTC, ...) come along into the new project. + for t in tracks(ls): + if t.tag == "AudioTrack" and t not in (pads_t, tempo_t) and t not in cleared: + for c in clips(t): + fr = c.find("SampleRef/FileRef") + if fr is not None and not (val(fr, "Path") or "").startswith(final_proj + os.sep): + files.keep_in_place(c, base_proj) + renumber(loc_parent) + setv(ls, "NextPointeeId", max(next_ptr, max_pointee(root) + 1)) + problems = check(root) + if problems: + raise RuntimeError("SET CHECK: " + problems[0]) + + status.write(step="COPYING %s" % svc.get("label", ""), copy_bytes=files.size()) + shutil.rmtree(building, ignore_errors=True) + make_project(building) + if not cfg.get("no_copy"): + files.copy(building, lambda d, n: status.write(step="COPYING %d/%d" % (d, n))) + save(root, os.path.join(building, name + ".als")) + if os.path.exists(final_proj): + raise RuntimeError("%s APPEARED WHILE BUILDING" % name) + os.rename(building, final_proj) + if files.missing: + log.append("%d AUDIO FILES MISSING" % len(files.missing)) + summary["missing"] = files.missing[:20] + summary["path"] = os.path.join(final_proj, name + ".als") + return summary + + +def check(root): + """What would make Live refuse the set (or misbehave): an empty list when it is sound.""" + ls = root.find("LiveSet") + out = [] + ids = [t.get("Id") for t in tracks(ls)] + if len(ids) != len(set(ids)): + out.append("TRACK IDS REPEAT") + for t in tracks(ls): + g = group_of(t) + if g != "-1" and g not in ids: + out.append("%s IN A GROUP THAT IS GONE" % tname(t)) + ptr = [e.get("Id") for e in root.iter() if is_pointee(e)] + if len(ptr) != len(set(ptr)): + out.append("AUTOMATION IDS REPEAT") + if ptr and max(int(p) for p in ptr) >= int(val(ls, "NextPointeeId")): + out.append("NEXT POINTEE ID TOO LOW") + scenes = len(list(ls.find("Scenes"))) + returns = len([t for t in tracks(ls) if t.tag == "ReturnTrack"]) + for t in tracks(ls): + lst = t.find("DeviceChain/MainSequencer/ClipSlotList") + if lst is not None and len(lst) != scenes: + out.append("%s HAS %d CLIP SLOTS" % (tname(t), len(lst))) + sends = t.find("DeviceChain/Mixer/Sends") + if sends is not None and t.tag != "ReturnTrack" and len(sends) != returns: + out.append("%s HAS %d SENDS" % (tname(t), len(sends))) + prev = None + for c in sorted(clips(t), key=lambda c: span(c)[0]): + s, e = span(c) + if e <= s: + out.append("%s: AN EMPTY CLIP AT %g" % (tname(t), s)) + if prev is not None and s < prev - 0.01: + out.append("%s: CLIPS OVERLAP AT %g" % (tname(t), s)) + prev = e + return out + + +# --- the job ---------------------------------------------------------------------------------- + +def projects_dir(cfg): + return os.path.expanduser(cfg["projects"]) + + +def weekend_pattern(cfg): + """"{date} {loc} Project" -> a pattern for the location's weekend sets (not "260909 NA FORGE Project").""" + f = re.escape(cfg.get("folder_format", "{date} {loc} Project")) + return "^" + f.replace(re.escape("{date}"), r"(\d{6})").replace(re.escape("{loc}"), re.escape(cfg["site_folder"])) + "$" + + +def newest_set(folder, before=None, pattern=r"^(\d{6}) "): + """The newest dated set in a location folder ("261025 NA Project"), before `before` (YYMMDD).""" + best = None + for n in os.listdir(folder) if os.path.isdir(folder) else []: + m = re.match(pattern, n) + if not m or n.endswith((".building", ".copying")) or (before and m.group(1) >= before): + continue + d = os.path.join(folder, n) + als = [os.path.join(d, x) for x in os.listdir(d) if x.endswith(".als") and not x.startswith(".")] if os.path.isdir(d) else [] + if als and (best is None or m.group(1) > best[0]): + best = (m.group(1), max(als, key=os.path.getmtime)) + return best[1] if best else None + + +def target(cfg, svc): + folder = os.path.join(os.path.expanduser(cfg.get("out_projects") or cfg["projects"]), svc["folder"]) + return folder, svc["name"] + + +def run(cfg): + status = Status({"state": "running", "started": time.time(), "step": "STARTING", "results": [], "log": []}) + log = status.data["log"] + try: + cfg["_pads"] = pads_index(cfg.get("pads_folder")) + cfg["_tunes"] = tune_index(cfg.get("tune_library")) + site_dir = os.path.join(projects_dir(cfg), cfg["site_folder"]) + todo = [] + for svc in cfg["services"]: + folder, name = target(cfg, svc) + d = os.path.join(folder, name) + if os.path.isdir(d): + status.data["results"].append({"name": name, "skipped": "THERE ALREADY"}) + continue + todo.append(svc) + if not todo: + status.write(state="done", step="NOTHING TO DO") + return + # 1. Last weekend's songs to Finished Songs (from the newest weekend set before the first new one). + first = min(s["date"] for s in todo) + prev = newest_set(site_dir, first, weekend_pattern(cfg)) + if prev and cfg.get("save_songs", True): + status.write(step="SAVING SONGS") + saved = export_songs(prev, cfg, status, log) + status.write(saved=saved) + for svc in sorted(todo, key=lambda s: s["date"]): + folder, name = target(cfg, svc) + base = newest_set(site_dir, svc["date"], weekend_pattern(cfg)) + if not base: + log.append("NO WEEKEND SET TO START %s FROM" % name) + continue + status.write(step="BUILDING %s" % svc["label"], base=os.path.basename(base)) + os.makedirs(folder, exist_ok=True) + notes = [] + res = build_set(svc, base, folder, name, cfg, status, notes) + res["notes"] = list(dict.fromkeys(notes)) + log.extend(res["notes"]) + status.data["results"].append(res) + status.write() + status.write(state="done", step="DONE") + except Exception as e: # noqa: BLE001 - the key shows what stopped it + import traceback + status.write(state="failed", step=str(e)[:60].upper(), trace=traceback.format_exc()[-2000:]) + + +def main(P): + if P.get("mode") == "run": + # Detached, so a hold returns at once; one job at a time. + try: + fd = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + if time.time() - os.path.getmtime(LOCK) < 3 * 3600: + print(json.dumps({"started": False, "busy": True})) + return + os.remove(LOCK) + fd = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.close(fd) + job = os.path.expanduser("~/Library/Caches/status-panel-start-job.json") + with open(job, "w") as f: + json.dump(P, f) + prog = os.path.expanduser("~/Library/Caches/status-panel-start.py") + src = globals().get("SOURCE") + if src is None: + with open(__file__) as f: + src = f.read() + with open(prog, "w") as f: + f.write(src) + subprocess.Popen(["/usr/bin/nice", "-n", "15", "/usr/bin/python3", prog, job], + stdout=open("/tmp/status-panel-start.log", "w"), stderr=subprocess.STDOUT, start_new_session=True) + print(json.dumps({"started": True})) + return + + +if __name__ == "__main__": + if len(sys.argv) > 1: # the detached job + try: + run(json.load(open(sys.argv[1]))) + finally: + try: + os.remove(LOCK) + except OSError: + pass diff --git a/config.json b/config.json index 47fa8b0..ba283ed 100644 --- a/config.json +++ b/config.json @@ -517,6 +517,13 @@ "kind": "ready", "short": "MOUNT", "label": "mount Location Assets again on the ProPresenter Mac (its keep-mounted agent), then look for the bundle again" + }, + "ready_start": { + "action": "start_week", + "kind": "ready", + "label": "build this week's Ableton sets (Sunday and special services) from Planning Center", + "never_auto": true, + "short": "START" } }, "flap": { @@ -1498,6 +1505,21 @@ 3, 7 ] + }, + { + "id": "rd_start", + "label": "START", + "pos": [ + 3, + 3 + ], + "checks": [ + { + "type": "ready", + "item": "start", + "fix": "ready_start" + } + ] } ], "page": 21, diff --git a/readiness.py b/readiness.py index 1986918..f85058b 100644 --- a/readiness.py +++ b/readiness.py @@ -30,6 +30,8 @@ snap.ready, which the READY page keys draw (check type "ready" in status_panel.p mics (a tap-to-confirm key) which mics this weekend needs, from PCO track_bundle every location has this weekend's Ableton project folder on the shared drive prop_bundle a ProPresenter bundle was saved to the locations' folder this week + start every service PCO lists this week (Sunday and special services such as First / Together + Tuesday) has its Ableton set; the hold builds the missing ones (ableton_start.py) What can be put right is put right by a hold (fix kind "ready", run_action below): re-sync the lineup from PCO, put Resolume back on its deck, copy the tracks to a location that @@ -1747,7 +1749,8 @@ def look(cfg, now=None, data_dir=None, sources=None): resolume=lambda: http_json(rc.get("resolume_url", "").rstrip("/") + "/api/v1/composition", timeout=6), macros=lambda paths: run_remote(cfg, rc["prop_host"], MACRO_SCRIPT % json.dumps({"paths": paths}), timeout=60), companion=lambda n: companion_page(cfg, n), live=lambda: live_look(cfg, rc), - hits=lambda al: light_hits(cfg, rc, al), + hits=lambda al: light_hits(cfg, rc, al), services=lambda: upcoming_services(rc, now), + start=lambda svcs: start_look(cfg, rc, svcs), prop_bundle=lambda plan: run_remote(cfg, rc["prop_bundle_host"], BUNDLE_SCRIPT % json.dumps( {"folder": prop_bundle_folder(rc, plan), "pattern": rc.get("prop_bundle_pattern", "*.probundle")}), timeout=30)) src.update(sources or {}) @@ -1772,6 +1775,7 @@ def look(cfg, now=None, data_dir=None, sources=None): ("timecode", rc.get("prop_url")), ("backgrounds", rc.get("prop_url")), ("lineup", rc.get("lineup_url")), ("mics", True), ("track_bundle", rc.get("locations")), ("prop_bundle", rc.get("prop_bundle_host")), + ("start", start_on(rc)), ("led_cross", rc.get("resolume_url") and rc.get("prop_host"))) if on], e) if plan is not None: prop = None @@ -1846,6 +1850,12 @@ def look(cfg, now=None, data_dir=None, sources=None): lines["lineup"] = eval_lineup(rc, plan, src["lineup"]()) except Exception as e: # noqa: BLE001 failed(["lineup"], e if isinstance(e, ReadyError) else ReadyError("LINEUP UNREACHABLE")) + if start_on(rc): + try: + services = src["services"]() + lines["start"] = eval_start(rc, services, src["start"](services), now) + except Exception as e: # noqa: BLE001 + failed(["start"], e) mics = Line() mics.info = mics_needed(rc, plan) lines["mics"] = mics @@ -1869,6 +1879,7 @@ class Poller(threading.Thread): super().__init__(name="ready", daemon=True) self.cfg, self.snap, self.stop, self.data_dir, self.is_live = cfg, snap, stop, data_dir, is_live self.interval = (cfg.get("ready") or {}).get("interval_seconds", 120) + IS_LIVE[0] = is_live def run(self): forced = True @@ -1891,6 +1902,187 @@ class Poller(threading.Thread): break +# --- START: the week's Ableton sets (ableton_start.py, run on the Tracks Mac) ---------------------------- +# The services come from PCO: this location's weekend service type, and every special one beside it (same +# parent folder in PCO: "Special Services" holds First / Together Tuesday, "Morning Worship", "Forge & +# Flourish"...), skipping youth, kids and classes. A plan counts when it has songs with a key. + +START_SOURCE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ableton_start.py") +START_TYPES = r"services|events|worship|forge|flourish" +START_SKIP = (r"youth|yth|kids|young adult|growth|baptism|guest|security|caf|preschool|nursery|toddler|elementary|" + r"academy|homeschool|audition|connect") +SERVICES = {} # the services START builds, kept a quarter of an hour (not asked of PCO every look) +IS_LIVE = [lambda: False] # the Poller's "a service is on": the hold copies gigabytes, never during one + +START_SURVEY = r''' +import json, os +P = json.loads(%r) +out = {} +for s in P["services"]: + d = os.path.join(os.path.expanduser(P["projects"]), s["folder"], s["name"]) + out[s["name"]] = {"exists": os.path.isdir(d) and any(n.endswith(".als") for n in os.listdir(d)), + "building": os.path.isdir(d + ".building")} +try: + st = json.load(open(os.path.expanduser("~/Library/Caches/status-panel-start.json"))) +except Exception: + st = None +print(json.dumps({"survey": out, "status": st})) +''' + + +def start_on(rc): + """START is where the sets are made: the location that sends its tracks to the others (or one set to).""" + on = rc.get("start") + return bool(rc.get("tracks_host") and rc.get("projects") and (on if on is not None else rc.get("locations"))) + + +def start_window(rc, now): + """Today up to the Thursday after this week's Sunday: on Monday that is Sunday and a First Tuesday.""" + today = datetime.fromtimestamp(now).replace(hour=0, minute=0, second=0, microsecond=0) + return time.mktime(today.timetuple()), week_start(now, rc.get("reset_day", "mon")) + 10 * 86400 + + +def plan_service(auth, st, plan_id, rc): + """(songs, salvation) of a plan: the band's songs with their PCO key and tempo, in order; Salvation is the + song after the Salvation moment (a TAG) if there is one, else the last song (as the sets do it).""" + items, inc = pco_get(auth, "/service_types/%s/plans/%s/items?include=arrangement&per_page=100" % (st, plan_id)) + bpm = {i["id"]: (i.get("attributes") or {}).get("bpm") for i in inc if i.get("type") == "Arrangement"} + skip = {norm(x) for x in rc.get("skip_songs", [])} + songs, sal, past = [], None, False + for it in items: + a = it.get("attributes") or {} + title = (a.get("title") or "").strip() + if re.search(r"salvation", title, re.I): + past = True + continue + if a.get("item_type") != "song" or a.get("service_position") not in ("during", "") or not a.get("key_name"): + continue + if norm(title) in skip: + continue + ref = ((it.get("relationships") or {}).get("arrangement") or {}).get("data") or {} + song = {"title": title, "key": a["key_name"], "bpm": bpm.get(ref.get("id"))} + if past or re.search(r"\bTAG\b", title, re.I): + sal = sal or song + continue + if not any(same(song["title"], x["title"]) and same_key(song["key"], x["key"]) for x in songs): + songs.append(song) + return songs, sal or (songs[-1] if songs else None) + + +def service_entry(rc, weekend, type_name, plan, t, songs, sal): + """Where a service's set goes: "NA/261101 NA Project", "FIRST TUESDAY/261103 NA First Tuesday Project".""" + d = datetime.fromtimestamp(t) + date, loc = d.strftime("%y%m%d"), rc.get("site_folder", "") + fmt = rc.get("folder_format", "{date} {loc} Project") + a = plan.get("attributes") or {} + title = " ".join(re.sub(r"[/:\\]", "-", (a.get("title") or type_name.split("|")[-1]).strip()).split()) + if weekend: + folder, name = loc, fmt.format(date=date, loc=loc) + else: + tues = re.search(r"tuesday", title + " " + type_name, re.I) + folder = rc.get("start_tuesday_folder", "FIRST TUESDAY") if tues else loc + name = fmt.format(date=date, loc="%s %s" % (loc, title)) + label = "%s %d/%d" % (d.strftime("%a").upper(), d.month, d.day) + return {"date": date, "folder": folder, "name": name, "label": label, "title": title or type_name, + "songs": songs, "salvation": sal, "service_type": plan.get("_st"), "plan_id": plan.get("id")} + + +def upcoming_services(rc, now, fresh=False): + """The services START builds sets for, soonest first.""" + lo, hi = start_window(rc, now) + key = (lo, int(now // 900)) + if not fresh and SERVICES.get("key") == key: + return SERVICES["list"] + auth, home = pco_login(rc) + try: + types, _ = pco_get(auth, "/service_types?per_page=100", pages=3) + except OSError as e: + raise ReadyError("PCO UNREACHABLE") from e + by_id = {x["id"]: x for x in types} + parent_of = lambda x: ((((x or {}).get("relationships") or {}).get("parent") or {}).get("data") or {}).get("id") + name_of = lambda x: ((x or {}).get("attributes") or {}).get("name") or "" + want = re.compile(rc.get("start_types", START_TYPES), re.I) + skip = re.compile(rc.get("start_skip_types", START_SKIP), re.I) + chosen = [home] + [x["id"] for x in types if x["id"] != home and parent_of(x) and parent_of(x) == parent_of(by_id.get(home)) + and want.search(name_of(x)) and not skip.search(name_of(x))] + chosen += [str(x) for x in rc.get("start_service_types", []) if str(x) not in chosen] + out = [] + for st in chosen: + try: + plans, _ = pco_get(auth, "/service_types/%s/plans?filter=future&order=sort_date&per_page=4" % st, pages=1) + except OSError as e: + raise ReadyError("PCO UNREACHABLE") from e + for p in plans: + t = parse_iso((p.get("attributes") or {}).get("sort_date")) + if t is None or not lo <= t < hi: + continue + songs, sal = plan_service(auth, st, p["id"], rc) + if songs: + out.append(service_entry(rc, st == home, name_of(by_id.get(st)), dict(p, _st=st), t, songs, sal)) + out.sort(key=lambda x: (x["date"], x["name"])) + SERVICES.update(key=key, list=out) + return out + + +def start_params(rc, services, mode): + projects = rc.get("projects", "") + return {"mode": mode, "projects": projects, "site_folder": rc.get("site_folder", ""), + "folder_format": rc.get("folder_format", "{date} {loc} Project"), "finished_folders": rc.get("finished_folders", []), + "pads_folder": rc.get("pads_folder") or os.path.join(os.path.dirname(projects.rstrip("/")), "PADS"), + "tune_library": rc.get("tune_library", ""), "routing": rc.get("routing") or {}, + "keep_groups": rc.get("start_keep_groups") or rc.get("ears_skip_groups") or ["MIDI GROUP", "RESI TRACKS"], + "pad_track": rc.get("pad_track", "AMBIENT PADS"), "tune_track": rc.get("tune_track", "WAVES TUNE"), + "lighting_track": rc.get("lighting_track", "LIGHTING CUES"), "sections_track": rc.get("sections_track", "SECTIONS"), + "guide_track": rc.get("guide_track", "+GUIDE"), "loopguide_track": rc.get("loopguide_track", "+LOOPGUIDE"), + "loop_cue_track": rc.get("loop_cue_track", "Loop Cue"), "tempo_track": rc.get("tempo_track", "TEMPO"), + "send_a": 1.0, "save_songs": rc.get("start_save_songs", True), "services": services} + + +def start_script(params): + """ableton_start.py with its job; it keeps its own source, to start itself detached on the Tracks Mac.""" + with open(START_SOURCE) as f: + src = f.read() + return "SOURCE = %r\n%s\nmain(json.loads(%r))\n" % (src, src, json.dumps(params)) + + +def start_look(cfg, rc, services): + return run_remote(cfg, rc["tracks_host"], START_SURVEY % json.dumps({"projects": rc.get("projects", ""), "services": [ + {"folder": x["folder"], "name": x["name"]} for x in services]}), timeout=30) + + +def eval_start(rc, services, look, now): + """START: every service this week has its set (built, or being built); what the last build left to do.""" + line = Line() + st = look.get("status") or {} + survey = look.get("survey") or {} + running = st.get("state") == "running" and now - (st.get("time") or 0) < 1800 + if running: + line.fault(WARN, (st.get("step") or "BUILDING")[:24], "START is building: %s" % st.get("step")) + elif st.get("state") == "failed" and now - (st.get("time") or 0) < 7 * 86400: + line.fault(ERR, "START FAILED", "the last build stopped: %s" % st.get("step")) + for svc in services: + got = survey.get(svc["name"]) or {} + if not got.get("exists") and not running: + line.fault(ERR, "NO SET %s" % svc["label"], "%s has no Ableton set yet (%s/%s): hold START to build it" % ( + svc["title"] or svc["label"], svc["folder"], svc["name"])) + if st.get("state") == "done" and now - (st.get("time") or 0) < 86400: + names = {x["name"] for x in services if (survey.get(x["name"]) or {}).get("exists")} + for res in st.get("results") or []: + if res.get("name") not in names: + continue + for note in res.get("notes") or []: + m = re.match(r"TRANSPOSE (.+?) \S+>\S+", note) + n = re.match(r"NEW SONG (.+?):", note) + if m: + line.fault(WARN, "CONSOLIDATE " + short(m.group(1), 12), "%s: %s" % (res["name"], note)) + elif n: + line.fault(WARN, "NEW SONG " + short(n.group(1), 12), "%s: %s" % (res["name"], note)) + else: + line.notes.append("%s: %s" % (res["name"], note)) + line.info = " ".join(x["label"] for x in services) or "NO SERVICES" + return line + + # --- Fixes (a hold on a READY key: fix kind "ready") ------------------------------------------------ def write_ltc(rc, songs, service_type, plan_id): @@ -1978,6 +2170,17 @@ def run_action(cfg, action, chk=None, sleep=time.sleep): return "NO AGENT" if r.get("no_agent") else "STILL NOT MOUNTED" if action == "light_timing": return fix_light_timing(cfg, rc) + if action == "start_week": + if IS_LIVE[0](): + return "SERVICE ON" + services = upcoming_services(rc, time.time(), fresh=True) + if not services: + return "NO SERVICES IN PCO" + r = run_remote(cfg, rc["tracks_host"], start_script(start_params(rc, services, "run")), timeout=60) + if r.get("busy"): + return "ALREADY BUILDING" + log.info("FIX START: building the sets for %s", ", ".join("%s (%s)" % (x["label"], x["name"]) for x in services)) + return True if r.get("started") else "DID NOT START" if action == "ableton_mix": _, _, changes = mix_plan(rc, live_look(cfg, rc)) if not changes: diff --git a/ready-page.companionconfig b/ready-page.companionconfig index 1b1f412f30b568884f4b153ccd4913653e85811f..c7685049482bfeeb6fe5787ab4cfa9c2c0e20627 100644 GIT binary patch literal 5040 zcmW-k2QZs|7{_ZAtr4ZI7B#A>wjySO+7z`ntywchZAmD9qtzC*sabo(C{k2nkJx*Q z+IuHRjKB6ichCLqeeT|S?)$v&_r5>YC<+RHk!dI5D^EKsTYr9ct9Nz+Hg2x&R$xar zu#Fqo-tpayopi_di>kHZD1wnK{XIRO*_LLcU;ZAcy?AmvpCwq6R$l$X=NMbH2kx)0 zPD<`Z5ph&hXo`t4Us^dhUF1EdFG)+M!<%vetR*+{)Z@TqAKXd2@2?$}Pb5a_qOki-H&L zJ4$a?4JI$Axsi3`>b;n4(q}*=U%(Lwx8jYw`@b8nVHXqoB>aUOqlzi@-$-=g?UVy@ zH%5~g4VG;~`%#zgvweP)&^SMRNAvmj*jV@oCn=97$&{DJmpE~Tz!48u?`5+I*Cb}l z*7~O09dqw=gSkbkVxfY&%%--RB=vd`MU0)VRwuJfQ3D9QGL8tmPl@Y!Q4KX`^ES*g zut9&*tM4^j->6UXq#K-YSX{cPg=k0UsXZ@(yB~&eV;!BdRDXypa$>a3K3ElBP%1ck z&%9d>`Y5Y$srmT8UFbe#45g=(AUpc|kZTGZMw$b4rFCvtsiI&Y+P|G4U})o5sx|-W@Wd$+AE$>{D^CLzT_4!tiP} zNzV2dQhZ(UQ!IbFX^HdMrI#oak*iknPHvRVDbvH(o$SkhPcMHVE$!HP8oY_QUOnn$ z`dXkN=u2skfhx7v`L^P0;J!(@MR@%cq9G+bp{_WGZ%L3(=H-%OW24O+445gC;aWel zM5#TY1oD;reVs9Cr39S zaQyV-(2_k$ioM5BY;KG*?_`R&SK!a&9}%%7phWv*+0xht?pL?x?nXus4<*a)9<@WH zKZlI~t|U$BtAKE!4bwInmeyQTB_ru_Zvg4j0zqQ%Bv8kj9Aj$6H6IT{M2*vCXf23 z^wJeB{XV3D*}!*!%E<7G{_!^YB&{U_rR?)9M-hKx0ql5RVhcPklu|q~j~?>IxY#MJ zQxIbqRU>I(tCLV#l72BF{3L~twiT5raZU@sB-p+zsC5yH54iZ{uopbAMu=%o@PyI& z_3AIlqs)GF*KWrl*v!)3{%r)FRx*z zrHzCR5;P=L;77g!`yfh#ub=0BKK%%m2Wk^Q+JWOkayU8M*RFX_+I_smdKt6$;}i;6 z!*+<^gMhnHH4*lt$37h8i0?bvR?zg4iOp#F`uji@vyJ>aiE(u_y^Z(A&7P7Mz2U^Z zj0uPdu!VP-M@Nhz=3?+-svuR+GLyOD)jh7$xZ#D9g%euQ{lLFa zkdj)no?l93q8qYKe6l=kx97oe>pc1-2n-1V5y&{oGCChSEdn6=i3wX6hNzY4hj-sT z5n4T(QWv*aHCgTS!Ge>`u$IthD3zRmM6n1b!sCnHh zCz{?Pnw|ksI{+N^D0WgJZcOhHd%Q1lcRN0f;WivTuW-C=6u^;6-ZppLMLgoEUaKx% zILgay%aeTwFe0Rmj_z9Rl?`4`G6=gjtwOXzv=iI%I1I$%;6`&74|9;^{`+dcOOk5x zYa-<2*l>J4l-pF&UDwttEklT>0zDz6o3V`-a@oxE10@>-EYwIAv>-CG*! z6P8Y~*1vf@lh}SSeFPoVDsbY6M1*L7d#0_M29Z%pjzG@4traJR)&njy*0NkUt|MKR ztMlO~VO-q}OiF!&3SmaFF^A{)75Bw1gK<9)6WcC!xRP@p#*8Mh>>{;pw_WCmwV*so zDthru#?w?Ak{Ik>b*>fSmPFckfCq%|LRyE5ius=f@Q!vY^uA8J=xo zjQYpSjCAu`#b+zU8@G;jc4xIrb?gWt5T_Mgq;4flMiE*jZTERB8LS=<1&C1>uyvgpK(8~aw9m_-Lue0qdEfBJs z#J#dS+&E)3IYvRp+K94L!x5zDDD~L@iIrkQkXkjtW+~Wy{BfPtf_nX4BUQu-tV|E; zlq-~VVL?`{eJZ32DNtYSq93lzU9Ry(;VUJMn$6r(r-CZWdsc z!b}*SyevDPtF($af9g~is$lJrKzcn?ahMe3p5V_M?N!l@o#tSO z0vU)qu{=lNhIpT`#_zf1L(*6dr#Btm96-kT2CuwXUd$a1j_`op--xhiN_DBkB~q5j zq~*E~4KJAPP@GW~oxc-<4TZ|!jTYhx^8_9e6NyaKg$KS>fe zW)l1U*=cL7OW#!ePq^z8Gs4b>CB${ghrROnPilE&_IRkd+Y_iKC{r-JLUUr( zbt)P0XtUPEi9f^f_DatTWM;jie4{32p}68?{JPm!$v?wW4hWtIo!|&m!UZS_>Q=%# zN_M5`_!56$;hvUZ}lcbc=FX0GYrY|fFzpagjyPC^JJhWS3^6K_`VFWsT;P+LS7I67hrA-%_iFXc0qF_vkg;$!^_ zG0}%f@*WhQtsanp4f9FwE7cz}emV<9$0l506V6bUWRhf3nwa22UI=4wG3zHyH3ypd zL5|O&rrZznt_E)t&43lLJN?=6_|lBsya0Xy=$n$8HsMoo=y_@5K;B$8-{3W)0|_mqn6LN5Uk_eKgpG4w=`y~^)yZjy zp>Bf#+F-+D!MI@D*TH$8JMIJvRqzSCsqr)1*-5}AQGM7VJSj{{bypee)-aQuLja3> z@8MaA*FlcuImvKThk>cdFrRi^e}vAL4-p!u4=VW83d93p1Kb-gEKy!VGghL%K*r87!?dS8k~IoydHjsef)$Lfcse1AiqaQZ$EMX=Tkggc@7HR zAc|znCl~sA_?r=AM@aI#e#mcX8oej+`R(a-MC?k^!qWx1J+yobkiIir{i;0rbBZC$;)%nU`Q~ei&0MIFQml8m4KKa zJ@Dup|3PLLuS&#%^vw@~QnQ3x%( zJ0lSEhJe_7W;|IZd1pk{hwI&4Z|~oqb$Ujp#s!|;u6OI?Q6W3n5&H%cO49ay?xb7U`?`Wq>>;e5Nr{Kvo{}U#G-mFir1NO& za{fkld0wPXH&wkha?c?=1@>qG8HO#L#Y>M;J$@gfXkRzT+&!C1UliD4OujMn{NS5Z zfrZAUn1Qv=MtnDIEeDrm%@wv25R*)5nhz z-rgoFiuv_XOMYnzdXfCS{Z8Nc(l?$C0NdHcev9b#{iy2u&v~uKIZTu7Hyac0wfzoq zojxb!d);~{+QpbyVC8>ZUjCKMIK%MY-BK4kYEi14ro^XGtLi$=H{`k;p63*3Tr9Tb zQfk=h1neAh(&K6lsj=AK(mUT2&KS6t3B_)$alKz}ypJ~FdprH>t?Qg5-qJ?TAtVH;Vo1OO5Dv@RFL#D%W{=Kdi_HhK~`XTjp{vuc^t8dq#1Zg2Qc8!G4~ik-{2i6{Ek8(>37 z-@An!Cw4cl9fXVT`!APt8AT@#%!FLQM)E0J+!h`4;;Y`?d7iJxj|mHdK#IO70T%PJ&E6w$6@X_THX8wqCB@ zUiRKzPOi>mYdOolzqDDS&X4cb#KqrLDDtTi&n^AX7OqB(%INwL^oq%V&fzt`&e!nJ zqv^+YSt*^Ws~~bCaeqp_@ujH;BlTvE7d{b_VwXN}6n!bpmHGudsX9kTKdd|7_ybD_ z!IP`4t~9%`$&-5Hd$4qQM|z(TNsn)?9f(yw&MGtCx0lk8Gx$k{Jdqg^osnvBQgXM{NlkwF_$t&mw5Tg*dSP{K8j}v z;iJF*;PLr^;ugUSh%2xRx($y*Mios*jgPxD7nSXLoWFpk~F{L86v{?>0R=HlH>m5C;G0Fx1nm1l>3MTK~R$#E(P z16eR39L+yr>@(lb_JEe=-_5LEsXho0DQA^2vX)29i+S!C1^cKs&l9Q#r9>-2omLV9 zwqKtl9CXYR`I&(<_SIzax4Uu7DPObJq6O%4u5WD%Z4$ab5Xfk7d6=R_VnsEp(xDm_ zp&-*4?T8;bTMrlB z85^=LFO_9Z)xC5pt$Z>4!u!YzoXR#3UZOUl{Cjy8!f`e6XT7_i=KWiY4C#Uzc-L*b{JICa@x#~wjX?wYkiX2LS|ZtI}_eVewW zvWh-YC9kdW>+yEZIL}8W3VEV!f0>0jVPBn^1(BHqndULn4YR^kip{6DIJUdPuo>P7 zSf7)**2jKO^f8N|6$;;OZ*C;cZoXRAf7m~8S0WHHuo|e#kZ7x(F|u-!y^NtiPEI$A zFbXql`?#s6_g(oMWYqWRZ+@`>#K@h+u9r@LSggD?9Uwb2dwkCo5Vstj8a;hrBtIIX zVF>0`dr+qdS$+2oFRpld&#Y_=b3D(gBwz-)1`fx^2n4GhdCu^eSjhI$KE8IG+W7;` zYyPFu{uom`;cUiN(}k_&j4Bf<&W$p8cZVhG@3^GiE)n`(bGPCBZXd^3heG0rM5&B^ znBu1~sI#!|FQzcoh(0Z8sp_7RZ)WFNoeHjXffLJCnT?9|?5q#v!0UV|dE$2rz$Q5k ziJZ@5mY#VkZb$#*Ah1k3Lz_!AuEYi?0p8#M$f<5X3&e4 zl3do$4X+wXM?hEXaY_Cj|IN{~ZM1JB_wi*ZaYc)}uI*F8iY5Li{ZX1{*a2N~F-iC< zEvMHvrfPSb{LMa)ce&4DSAXsZAEQm zJo%_d*uZ9CSBlT=ovtpiH$b>L;F{?h%8m!@*6+mPGvx`iVzVWOq&6q|L={ozpVB`eQPBTN%*(;X<#z%W5(8%So`?a2tB z2Q;O0{0E~-EyA>;DzyB45XQZXBF3>A-Ug1<4z{doM_}A-1YtZF7axQhr+d`UWuvWy zG~oWu`!Kvqu0IQ8#%);4#ty&9z{9{ZAyPBZVS8X@_Kv8(@wl*^)K21cncctSUn>Wv zEa|^22SbIP%1Z>0rc&qc8Bt=0+Vq!Anq$GeYG2E4zyo02+Ov9lc6WBS$2z$M z1VEAoKPIs6e;&TPVv^nnxrCOA%8soWYarsFEf%)q^u6VJ1=r2mz;&=M3zB@2L^i1) z%IE@gX=R?20+QDfsed^c&h^i3p7Gf{p+9XDu;!?{LlfJ?Iw|E%fVvE)y{O}(6P2E zU^p#g?(HTpBLJ_O-s3O(u;Zzs=y-aH9ul{SJ}`av-q22&A=I0H@8{yJJwrQ$wCKX7 zzPOUGeIES=)G%=01tC=ol}Ur&EhL^%Y&R?_)ZjUwKUe3vvBTFp6I4a=-dGCW*RefQ zOrz;~tVbU|z|3U*YzDlMlajx0qeqH@uO!PQ#+zb;meX1Uyob4FL~*rmK2FS4a~ivg z=2lyeE4H5Qtz{Hmb#AnJ5oxvm`ee~=;c3C(Ctl^{gwz%p#dsv%-R+bP3CSK zm}n~op?w8P7Sqd%mV(kaM2)XT2GqsrnxZpKQmMjz?r_qV6F=l0>Sg3gTG|nRnuu)* zxOpn?21N?WSX?#fdguqnHdW5ba9F3UznPyss}94tzigu91wkJcdNfqvzQe|wt)9fHm}9`}YtPgVhZkH>)jEgt>x9XtPSi)=#F46(9#X8xX1Q5P?^) zAC(eAb>eh6KCWY98X$KYZH8Ec( zHmBpBv?+W^+@wWoyeS@!G=xiuNBe-lnmpD!unAPnKsk`TbXT1f+w&E2gU)t>nAqus_rspCb0b;S^*_15C?1^aYl)S&qHDBXnrhRJ+O>panY9W= zRj3Uu?OPpeFyBBqYd5RPznFe_qfDE}GU`*S*`SILGPJR>WGb}!b%H?+G0}wYiJ)oo z_bHssX!NE^cWiqShh)M0!<~NZz0C2&*}M24j~B^3;%=Qm9;C6CeU{P(rJSh4YDrF6 zr;Zdck>!Wki4*>haKu(gLW%VIfNLH(uYt>k= zg7H1@ed`g6u)R3>05O_mog=j4B!eG3Hd!q4x42p*_ew4FS9WFvE-mC#CTYGjRXOCu&dc2aSL4a{@^o;(FnOPOA3Kq+5s(c9* z1w}RedqnP!f_q}BSngu3n_n>`ZQpB!X_ z`*EI1-c=!+#{ER5sx@;XqQ{zW!NARBy)xV^E->whgp)aU!=IL@IyvEKkdAyy>9MBswZGS z9S107_myot3D$)y47Jy1^XH36qJ{i0)MyMqw)keA3+@pB$#RpaV81N5BfhEkF!NyyG) zB%I<~E8|uRJ@x8zyb+|p+*ni8eI_#nKz3N^c!E^MAoEqy0pm3-I=AIvZYHP}Q+~6g zwCB5t`RAis`KN}H7q%mVf}DLRfLy304}M6ZnVzkKO8Z^%9iDZGh$o?qS-h{F1v3Dj z2a$b>o+JNvxXf~W8>+lk!pUyaqxkUaNUo{+a6f3uj}pe;?ET-vZ4jduT3*QB}PaK)!`c zs4Hu_dzR|6(DvYL9?g%j<$`H#5qEB+lfKF} z)c*1|o_fOS$5uB0gOoT=NgB&?DuBu9YV5#Y;QOYvS|kp&GE`(0T$HWf&}SxnVKrML z$HBgHipkFKq)~U%bjRdJ{EocfU$mS(u#70UKJFDT9cY(E8F|pJ1g`ufGkeTE8(Li0 zzTUnLtqEP^CEF)Foyx$-{+ynvi{DK#uNUpgigeMS?6RU?|A1K&uG4>V8YFlPtYXnv z2oR1pYtmKvI>T9dkyqZH)-VRWT9un6Z>MR0tegPbdqkyj^D5P+_6-^dsj3HW+4z5j zCy(C|fCW9h@SDl)Wjg70-By4Pu`PrZbSmmbojjq#CuQRj?@)=08{xM766ycvZ4fwZ zd_W6f!BIWw|I>k%kete;aDMbVR9lxKaX9B(5@xmMdpa2Wl%d2-xmG#Si5;b^qI)v; zdg3oTv-ndz%XAq-&7GdHyW}xGC(WNBs@HKMpK(=@8T$i} ztlaQr*Nj|hvyL5UUHyylFX!QKW#e%UmFn?zWeMzVsj&-ff~62I9be7{+pZSDpI&>!j?#?p=5=XsPDV*=Bd;uy-XC*=a|8J!4GIlv7~}+i$i_Q(K;1 z|H8hojFRYBpH`~r2FZZ^&aSJTko06c{zmcVcD1HmXq&2yob)vJ5JHJkn@5}HD|G?z zl7D$`IlVGLAYB!~QN?w0C~xMUKH`YvxIDCMzH zD~6@*IV@p|;X&gDr*YkD!eisOx%#nHD2N`TzlmjT?j0)PktN3N3I))N+sq$5^9l0_ zbHMa?0b^FT~%Fp$gQgha7`Qe9}fqkOq8Z=QU+YC zOL!0-E`Y{xcPdq5!XSZYY1hdLf~_~JB: CONSOLIDATE IN LIVE", "NEW SONG GRACE: BUILD IT BY HAND"]}]}}, MONDAY + 60) + self.assertEqual([m for _, m, _ in line.faults], ["CONSOLIDATE THE LORD IS.", "NEW SONG GRACE"]) + line = rd.eval_start(RC, svc, {"survey": both, "status": None}, MONDAY) + self.assertEqual((line.faults, line.info), ([], "SUN 11/1 TUE 11/3")) + line = rd.eval_start(RC, svc, {"survey": {}, "status": {"state": "running", "time": MONDAY, "step": "COPYING 3/40"}}, MONDAY + 5) + self.assertEqual([m for _, m, _ in line.faults], ["COPYING 3/40"]) # no "NO SET" while it builds + + def test_only_where_tracks_are_made(self): + self.assertTrue(rd.start_on(RC)) + self.assertFalse(rd.start_on(dict(RC, locations=[]))) # a location that receives + self.assertTrue(rd.start_on(dict(RC, locations=[], start=True))) + + +# --- a small Live set, written the way Live 11 writes one --------------------------------------------------- + +def v(tag, value): + return '<%s Value="%s" />' % (tag, value) + + +PTR = [100] + + +def ptr(): + PTR[0] += 1 + return PTR[0] + + +def mixer(sends=1): + holders = "".join('%s' + '%s' + % (i, v("Manual", "0.0003162277571"), ptr(), ptr(), v("Active", "true")) for i in range(sends)) + return ('%s%s' + % (holders, v("Manual", "1"), ptr())) + + +def routing(upper="Master", lower="", target="AudioOut/Master"): + return "%s%s%s" % (v("Target", target), v("UpperDisplayString", upper), v("LowerDisplayString", lower)) + + +def slots(n=2): + return "%s" % "".join('' % i for i in range(n)) + + +def audio_clip(start, end, name, path, rel="", warped=False, secs=None, size=1000): + secs = secs if secs is not None else end - start + loop = "%s%s%s%s%s%s%s" % (v("LoopStart", 0), v("LoopEnd", secs), v("StartRelative", 0), v("LoopOn", "false"), + v("OutMarker", secs), v("HiddenLoopStart", 0), v("HiddenLoopEnd", secs)) + fr = "%s%s%s%s%s" % (v("RelativePathType", 3 if rel else 0), v("RelativePath", rel), v("Path", path), + v("OriginalFileSize", size), v("OriginalCrc", 1)) + return ('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s' + % (start, v("CurrentStart", start), v("CurrentEnd", end), loop, v("Name", name), v("Disabled", "false"), + v("IsWarped", "true" if warped else "false"), fr, v("LastModDate", 0), v("DefaultDuration", 100), v("DefaultSampleRate", 48000), + v("WarpMode", 0), v("ComplexProFormants", 100), v("PitchCoarse", 0), v("Fade", "true"), v("Sync", "true"))) + + +def midi_clip(start, end, name, notes=(), program=None): + kt = ('%s%s' + % ("".join('' % n for n in notes), v("MidiKey", 66)) + if notes else "") + return ('%s%s%s%s%s%s%s%s' + % (start, v("CurrentStart", start), v("CurrentEnd", end), v("LoopStart", 0), v("LoopEnd", end - start), v("Name", name), + v("Disabled", "false"), kt, v("ProgramChange", program if program is not None else -1))) + + +def track(kind, id_, name, group="-1", clips="", out=("Master", "", "AudioOut/Master")): + seq = ("%s" if kind == "AudioTrack" else + "%s") % clips + if kind == "GroupTrack": + return ('%s%s%s' + '%s%s' % (id_, v("EffectiveName", name), v("UserName", name), v("TrackGroupId", group), + mixer(), routing(*out))) + return ('<%s Id="%s">%s%s%s%s%s%s%s' + '' + % (kind, id_, v("EffectiveName", name), v("UserName", name), v("TrackGroupId", group), mixer(), routing(*out), slots(), seq, kind)) + + +def live_set(tracks_xml, locators, tempo=72): + locs = "".join('%s%s%s%s%s' % ( + i, v("Time", t), v("Name", n), v("Annotation", ""), v("PersistentKeyString", k), v("NoteOrController", -1)) + for i, (t, n, k) in enumerate(locators)) + return ('%s' + '%s%s%s%s%s%s' + '%s' + '%s' + '' + '%s' + % (v("NextPointeeId", 5000), tracks_xml, v("EffectiveName", "A-IN EAR MIX"), v("UserName", "A-IN EAR MIX"), v("TrackGroupId", -1), + mixer(0), routing(), v("Manual", tempo), v("PointeeId", 8), tempo, locs)) + + +def write_als(path, xml): + os.makedirs(os.path.dirname(path), exist_ok=True) + with gzip.open(path, "wb") as f: + f.write(xml.encode()) + + +def write_wav(path, seconds=2.0, rate=8000): + os.makedirs(os.path.dirname(path), exist_ok=True) + w = wave.open(path, "wb") + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(b"\0\0" * int(seconds * rate)) + w.close() + + +class TestBuild(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir) + d = self.dir + self.proj = os.path.join(d, "Projects") + base = os.path.join(self.proj, "NA", "261025 NA Project") + write_wav(os.path.join(base, "Samples/Imported/Piano.wav")) + write_wav(os.path.join(base, "Samples/Imported/D# - WARM - CHURCHFRONT PADS.wav"), 600) + write_wav(os.path.join(base, "Samples/Imported/LTC.wav")) + write_wav(os.path.join(base, "Samples/Imported/Guide.wav")) + p = lambda n: os.path.join(base, "Samples/Imported", n) + ext = ("Ext. Out", "5/6 Keys", "AudioOut/External/S2") + last_week = ( + track("AudioTrack", 57, "TEMPO", clips=audio_clip(0, 1132, "143.00 BPM", "Samples/Recorded/x.aif", warped=True)) + + track("AudioTrack", 58, "LTC", clips=audio_clip(0, 4000, "LTC", p("LTC.wav"), "Samples/Imported/LTC.wav", True)) + + track("MidiTrack", 59, "SECTIONS", clips=midi_clip(0, 196, "CALL TO WORSHIP") + midi_clip(196, 236, "INTRO")) + + track("GroupTrack", 60, "MIDI GROUP") + + track("MidiTrack", 61, "Loop Cue", "60", midi_clip(138, 138.5, "", [0]) + midi_clip(699, 700, "") + midi_clip(1120.5, 1121, "", [0])) + + track("MidiTrack", 62, "WAVES TUNE", "60", midi_clip(0, 1, "Ebmaj", program=7) + midi_clip(196, 197, "Ebmaj", program=7)) + + track("MidiTrack", 63, "LIGHTING CUES", "60", midi_clip(196, 900, "LIGHTING CUES 56")) + + track("GroupTrack", 64, "RESI TRACKS") + + track("AudioTrack", 65, "+GUIDE", clips=audio_clip(196, 900, "Guide", p("Guide.wav"), "Samples/Imported/Guide.wav", True)) + + track("AudioTrack", 66, "+LOOPGUIDE") + + track("AudioTrack", 67, "AMBIENT PADS", clips=audio_clip(0, 237, "D# - WARM - CHURCHFRONT PADS", p("D# - WARM - CHURCHFRONT PADS.wav"), + "Samples/Imported/D# - WARM - CHURCHFRONT PADS.wav", secs=99.4)) + + track("GroupTrack", 70, "OLD SONG") + + track("AudioTrack", 71, "21-Piano", "70", audio_clip(196, 900, "Piano", p("Piano.wav"), "Samples/Imported/Piano.wav", secs=295.1), ext) + + track("AudioTrack", 72, "22-EG 1", "70", audio_clip(196, 900, "EG 1", p("Piano.wav"), "Samples/Imported/Piano.wav", secs=295.1), + ("Ext. Out", "7/8 GTRS", "AudioOut/External/S3"))) + locators = [(0, "1- CALL TO WORSHIP ", "1"), (76, "LOOP", ""), (196, "2 - OLD SONG- Eb", "2"), (1090, "LOOP", ""), + (1132, "3- SALVATION- Eb", "3"), (1500, "LOOP", "")] + write_als(os.path.join(base, "261025 NA Project.als"), live_set(last_week, locators, 143)) + # Finished Songs: one song in E (to be used in E and, transposed, in B). + self.fin = os.path.join(d, "Finished Songs") + song = os.path.join(self.fin, "GRACE [E]") + write_wav(os.path.join(song, "Samples/Processed/Consolidate/Keys [x].aif".replace(".aif", ".wav"))) + write_wav(os.path.join(song, "Samples/Processed/Consolidate/Guide [2026-01-01 000000].wav")) + c = lambda n: os.path.join(song, "Samples/Processed/Consolidate", n) + song_tracks = ( + track("GroupTrack", 0, "GRACE") + + track("MidiTrack", 1, "2-MIDI", "0", midi_clip(1992, 2024, "INTRO") + midi_clip(2024, 2056, "VERSE 1")) + + track("MidiTrack", 2, "3-MIDI", "0", midi_clip(1992, 2300, "LIGHTING CUES 47")) + + track("AudioTrack", 3, "4-Guide [2026-01-01 000000]", "0", audio_clip(1992, 2300, "Guide", c("Guide [2026-01-01 000000].wav"), + "Samples/Processed/Consolidate/Guide [2026-01-01 000000].wav", True)) + + track("AudioTrack", 4, "5-Keys 1 [x]", "0", audio_clip(1992, 2368, "Keys 1", c("Keys [x].wav"), "Samples/Processed/Consolidate/Keys [x].wav", + secs=313.3333333), ext)) + write_als(os.path.join(song, "GRACE.als"), live_set(song_tracks, [], 143)) + self.pads = os.path.join(d, "PADS") + for k in ("E", "B", "D#"): + write_wav(os.path.join(self.pads, "%s - WARM - CHURCHFRONT PADS.wav" % k), 600) + self.tune = os.path.join(d, "Waves Tune") + for name, prog in (("Emaj", 4), ("Bmaj", 11), ("Ebmaj", 7)): + write_als(os.path.join(self.tune, name + ".alc"), '%s' % v("ProgramChange", prog)) + A.STATUS = os.path.join(d, "status.json") + self.cfg = {"projects": self.proj, "site_folder": "NA", "folder_format": "{date} {loc} Project", + "finished_folders": [self.fin], "pads_folder": self.pads, "tune_library": self.tune, + "routing": {"KEYS 1": ["Ext. Out", "5/6 Keys"], "EG 1": ["Ext. Out", "7/8 GTRS"]}, + "keep_groups": ["MIDI GROUP", "RESI TRACKS"], "pad_track": "AMBIENT PADS", "tune_track": "WAVES TUNE", + "lighting_track": "LIGHTING CUES", "sections_track": "SECTIONS", "guide_track": "+GUIDE", + "loopguide_track": "+LOOPGUIDE", "loop_cue_track": "Loop Cue", "tempo_track": "TEMPO", "send_a": 1.0, + "services": [{"date": "261101", "folder": "NA", "name": "261101 NA Project", "label": "SUN 11/1", + "songs": [{"title": "Grace", "key": "E", "bpm": 72}, {"title": "Grace", "key": "B", "bpm": 72}, + {"title": "Brand New", "key": "D", "bpm": 120}], + "salvation": {"title": "Grace TAG", "key": "B", "bpm": 72}}]} + + def run_job(self): + A.run(self.cfg) + with open(A.STATUS) as f: + return json.load(f) + + def test_builds_the_weeks_set(self): + st = self.run_job() + self.assertEqual(st["state"], "done", st.get("trace")) + # 1. Last week's song went to Finished Songs, with its markers, lights and guide inside the group. + self.assertEqual(st["saved"], ["OLD SONG [Eb]"]) + old = A.load(os.path.join(self.fin, "OLD SONG [Eb]", "OLD SONG.als")).find("LiveSet") + self.assertEqual([A.tname(t) for t in A.tracks(old) if t.tag != "ReturnTrack"], + ["OLD SONG", "SECTIONS", "LIGHTING CUES", "Guide", "21-Piano", "22-EG 1"]) + self.assertTrue(os.path.exists(os.path.join(self.fin, "OLD SONG [Eb]", "Samples/Imported/Piano.wav"))) + # 2-6. The new set. + path = os.path.join(self.proj, "NA", "261101 NA Project", "261101 NA Project.als") + root = A.load(path) + ls = root.find("LiveSet") + self.assertEqual(A.check(root), []) + locs = sorted((A.fval(l, "Time"), A.val(l, "Name"), A.val(l, "Mapping/PersistentKeyString")) for l in ls.findall("Locators/Locators/Locator")) + names = [(n, k) for _, n, k in locs if not n.startswith("LOOP")] + self.assertEqual(names, [("1- CALL TO WORSHIP - E", "1"), ("2- GRACE - E", "2"), ("3- GRACE - B", "3"), + ("4- BRAND NEW - D", "4"), ("5- SALVATION - B", "5")]) + by = {A.tname(t): t for t in A.tracks(ls)} + # The 376-beat song pushed the 1090 loop along: the loop starts after the song ends. + loops = [t for t, n, _ in locs if n.startswith("LOOP")] + self.assertEqual(loops[:2], [76.0, 1090.0]) + self.assertEqual([A.span(c)[0] for c in A.clips(by["Loop Cue"])][:2], [138.0, 1120.5]) + self.assertNotIn(699.0, [A.span(c)[0] for c in A.clips(by["Loop Cue"])]) # an empty cue does nothing + tunes = [(A.span(c)[0], A.val(c, "Name"), A.val(c, "ProgramChange")) for c in A.clips(by["WAVES TUNE"])] + self.assertEqual([t[1:] for t in tunes], [("Emaj", "4"), ("Emaj", "4"), ("Bmaj", "11"), ("Bmaj", "11")]) # no D cue in the library + pads = [(A.val(c, "Name")[:2], A.span(c)) for c in A.clips(by["AMBIENT PADS"])] + self.assertEqual([p[0] for p in pads], ["E ", "E ", "B ", "B "]) # no D pad: named in the notes + self.assertEqual([A.tname(t) for t in A.tracks(ls) if t.tag == "GroupTrack" and A.group_of(t) == "-1"], + ["MIDI GROUP", "RESI TRACKS", "GRACE", "GRACE"]) + g2 = [t for t in A.tracks(ls) if A.tname(t) == "GRACE"][1] + keys = [t for t in A.family(ls, g2) if t.tag == "AudioTrack"][0] + c = A.clips(keys)[0] + self.assertEqual((A.val(c, "IsWarped"), A.val(c, "WarpMode"), A.val(c, "ComplexProFormants"), A.val(c, "PitchCoarse")), + ("true", "6", "0", "-5")) + self.assertEqual(A.val(keys, "DeviceChain/Mixer/Sends/TrackSendHolder/Send/Manual"), "1") + self.assertEqual(A.val(keys, "DeviceChain/AudioOutputRouting/Target"), "AudioOut/External/S2") + self.assertEqual([A.val(c, "Name") for c in A.clips(by["SECTIONS"])], ["CALL TO WORSHIP", "INTRO", "VERSE 1", "INTRO", "VERSE 1"]) + self.assertEqual(len(A.clips(by["+GUIDE"])), 2) + self.assertEqual(A.clips(by["+LOOPGUIDE"]), []) + fr = c.find("SampleRef/FileRef") + self.assertEqual(A.val(fr, "RelativePath"), "Samples/Imported/GRACE [E]/Keys [x].wav") + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(path), "Samples/Imported/GRACE [E]/Keys [x].wav"))) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(path), "Samples/Imported/LTC.wav"))) + tempo = [(A.span(c)[0], A.val(c, "Name")) for c in A.clips(by["TEMPO"])] + self.assertEqual(tempo[0], (0.0, "72.00 BPM")) + self.assertIn((tempo[-1][0], "72.00 BPM"), tempo) + notes = st["results"][0]["notes"] + self.assertIn("TRANSPOSE GRACE E>B: CONSOLIDATE IN LIVE", notes) + self.assertIn("NEW SONG BRAND NEW: BUILD IT BY HAND", notes) + self.assertIn("NO PAD IN D", notes) + self.assertFalse([n for n in os.listdir(os.path.join(self.proj, "NA")) if n.endswith(".building")]) + self.assertTrue(os.path.isdir(os.path.join(os.path.dirname(path), "Ableton Project Info"))) # else Live finds no audio + self.assertTrue(os.path.isdir(os.path.join(self.fin, "OLD SONG [Eb]", "Ableton Project Info"))) + + def test_a_set_that_is_there_is_left_alone(self): + os.makedirs(os.path.join(self.proj, "NA", "261101 NA Project")) + st = self.run_job() + self.assertEqual(st["step"], "NOTHING TO DO") + self.assertEqual(os.listdir(os.path.join(self.proj, "NA", "261101 NA Project")), []) + + def test_an_unchanged_song_is_not_saved_again_and_a_changed_one_parks_the_old(self): + self.run_job() + shutil.rmtree(os.path.join(self.proj, "NA", "261101 NA Project")) + st = self.run_job() + self.assertEqual(st["saved"], []) + self.assertFalse(os.path.exists(os.path.join(self.fin, "zOLD SONG [Eb]"))) + # A re-consolidated stem (new file) is a new version: the old folder is parked as z. + base = os.path.join(self.proj, "NA", "261025 NA Project", "261025 NA Project.als") + xml = gzip.open(base).read().decode() + i = xml.index("21-Piano") + xml = xml[:i] + xml[i:].replace('Value="1000"', 'Value="2000"', 1) + write_als(base, xml) + shutil.rmtree(os.path.join(self.proj, "NA", "261101 NA Project")) + st = self.run_job() + self.assertEqual(st["saved"], ["OLD SONG [Eb]"]) + self.assertTrue(os.path.isdir(os.path.join(self.fin, "zOLD SONG [Eb]"))) + + def test_a_changed_song_replaces_the_version_it_was_made_from(self): + # Last week's "OLD SONG" group was made from "OLD SONG LIVE [Eb]", then edited (a stem swapped). + song = os.path.join(self.fin, "OLD SONG LIVE [Eb]") + write_wav(os.path.join(song, "Samples/Piano.wav")) + write_wav(os.path.join(song, "Samples/Keys.wav")) + write_als(os.path.join(song, "OLD SONG LIVE.als"), live_set( + track("GroupTrack", 0, "OLD SONG LIVE") + + track("AudioTrack", 1, "Piano", "0", audio_clip(0, 10, "Piano", os.path.join(song, "Samples/Piano.wav"), "Samples/Piano.wav")) + + track("AudioTrack", 2, "Keys", "0", audio_clip(0, 10, "Keys", os.path.join(song, "Samples/Keys.wav"), "Samples/Keys.wav")), [])) + st = self.run_job() + self.assertEqual(st["saved"], ["OLD SONG LIVE [Eb]"]) + self.assertTrue(os.path.isdir(os.path.join(self.fin, "zOLD SONG LIVE [Eb]"))) + self.assertTrue(os.path.exists(os.path.join(self.fin, "OLD SONG LIVE [Eb]", "OLD SONG LIVE.als"))) + self.assertFalse(os.path.exists(os.path.join(self.fin, "OLD SONG [Eb]"))) + + def test_titles(self): + self.assertTrue(A.close_titles("CAN'T HELP BUT PRAISE", "Can't Help But Praise LM")) + self.assertFalse(A.close_titles("PRAISE", "Owe You Praise")) + self.assertEqual(A.semitones("E", "B"), -5) + self.assertEqual(A.semitones("G", "A"), 2) + self.assertEqual(A.pick_version([("/f/GRACE [E]", "GRACE", "E"), ("/f/GRACE [B]", "GRACE", "B")], "Grace", "B")[0][2], "B") + + def test_audio_info(self): + p = os.path.join(self.dir, "x.wav") + write_wav(p, 3.0, 8000) + self.assertEqual(A.audio_info(p), (24000, 8000)) + + +if __name__ == "__main__": + unittest.main()