srctree

Andrew Kelley parent 5982b953 af9a47ba
implement saving waveforms to the db

README.md added: 66, removed: 13, total 53
@@ -71,9 +71,7 @@ This extension makes it easy to debug Zig WebAssembly code:
* implement queue item sort keys
* when scanning the file system, detect files that are in the database but have
been deleted from the file system
* implement scanning for true duration, loudness, and waveforms
* apply loudness information to playback
* use the thread pool when scanning the music directory
* example that transcodes a list of input files
* Serving both https and http on the same port
* Automatic SSL Certificate obtaining and renewal
 
filename was Deleted added: 66, removed: 13, total 53
 
build.zig added: 66, removed: 13, total 53
@@ -44,22 +44,37 @@ pub fn build(b: *std.Build) void {
});
const libsoundio = soundio_dep.artifact("soundio");
 
const twi_mod = b.addModule("Twi", .{
.root_source_file = b.path("Twi/Twi.zig"),
.target = target,
.optimize = optimize,
});
 
const player = b.addModule("player", .{
.root_source_file = b.path("player/root.zig"),
.target = target,
.optimize = optimize,
.imports = &.{
.{ .name = "Twi", .module = twi_mod },
},
});
 
const shared_server = b.addModule("shared", .{
.root_source_file = b.path("shared/root.zig"),
.target = target,
.optimize = optimize,
.imports = &.{
.{ .name = "Twi", .module = twi_mod },
},
});
 
const shared_wasm = b.addModule("shared", .{
.root_source_file = b.path("shared/root.zig"),
.target = wasm_target,
.optimize = wasm_optimize,
.imports = &.{
.{ .name = "Twi", .module = twi_mod },
},
});
 
player.addImport("av", ffmpeg_dep.module("av"));
 
player/root.zig added: 66, removed: 13, total 53
@@ -8,7 +8,7 @@ pub const av = @import("av");
pub const SoundIo = @import("SoundIo");
pub const acoustid = @import("acoustid.zig");
pub const musicbrainz = @import("musicbrainz.zig");
pub const Twi = @import("Twi.zig");
pub const Twi = @import("Twi");
 
// Used by MusicBrainz for rate-limiting and contacting us if
// there is misbehaving code in the wild.
 
server/main.zig added: 66, removed: 13, total 53
@@ -10,6 +10,7 @@ const protocol = @import("shared").protocol;
const WebSocket = @import("WebSocket.zig");
const StaticHttpFileServer = @import("StaticHttpFileServer");
const Directory = std.Build.Cache.Directory;
const Twi = @import("Twi");
 
const usage =
\\Usage: groovebasin [options]
@@ -559,11 +560,14 @@ const Scan = struct {
const basename_z = try arena.dupeZ(u8, entry.name);
scan.new_count += 1;
 
try db.resizeWaveformArray(gpa, gop.index + 1);
 
scan.mutex.unlock();
defer scan.mutex.lock();
 
rcd_parent.ref();
const file_index: Db.File.Index = @enumFromInt(gop.index);
 
rcd_parent.ref();
scan.thread_pool.spawnWg(&scan.wait_group, analyzeFile, .{
scan, rcd_parent, basename_z, file_index,
});
@@ -594,7 +598,7 @@ const Scan = struct {
};
defer file.close();
 
var analysis = file.analyze(1920) catch |err| {
var analysis = file.analyze(Db.waveform_width) catch |err| {
std.log.err("unable to analyze '{s}': {s}", .{ basename_z, @errorName(err) });
return;
};
@@ -603,6 +607,8 @@ const Scan = struct {
scan.mutex.lock();
defer scan.mutex.unlock();
 
db.writeWaveformReserved(file_index, analysis.twi);
 
const metadata = fileMetaData(gpa, db, file) catch |err| {
std.log.err("unable to extract metadata from '{s}': {s}", .{ basename_z, @errorName(err) });
return;
@@ -854,6 +860,8 @@ fn saveDb(db: *const Db, install_directory: Directory, db_path: []const u8) !voi
const directories = std.mem.sliceAsBytes(db.directories.keys());
const albums = std.mem.sliceAsBytes(db.albums.keys());
const string_bytes = db.string_bytes.items;
const waveform_bytes = std.mem.sliceAsBytes(db.waveforms.items);
 
const header: DbFileHeader = .{
.magic = .v1,
.files_len = @intCast(db.files.entries.len),
@@ -868,6 +876,7 @@ fn saveDb(db: *const Db, install_directory: Directory, db_path: []const u8) !voi
.{ .base = directories.ptr, .len = directories.len },
.{ .base = albums.ptr, .len = albums.len },
.{ .base = string_bytes.ptr, .len = string_bytes.len },
.{ .base = waveform_bytes.ptr, .len = waveform_bytes.len },
};
 
var atomic_file = try install_directory.handle.atomicFile(db_path, .{});
@@ -895,20 +904,23 @@ fn loadDb(gpa: Allocator, db: *Db, install_directory: Directory, db_path: []cons
try db.directories.entries.resize(gpa, header.directories_len);
try db.albums.entries.resize(gpa, header.albums_len);
try db.string_bytes.resize(gpa, header.string_bytes_len);
try db.resizeWaveformArray(gpa, header.files_len);
 
const files = std.mem.sliceAsBytes(db.files.keys());
const directories = std.mem.sliceAsBytes(db.directories.keys());
const albums = std.mem.sliceAsBytes(db.albums.keys());
const string_bytes = db.string_bytes.items;
const waveforms = std.mem.sliceAsBytes(db.waveforms.items);
 
var iovecs = [_]std.posix.iovec{
.{ .base = files.ptr, .len = files.len },
.{ .base = directories.ptr, .len = directories.len },
.{ .base = albums.ptr, .len = albums.len },
.{ .base = string_bytes.ptr, .len = string_bytes.len },
.{ .base = waveforms.ptr, .len = waveforms.len },
};
const amt_read = try file.readvAll(&iovecs);
const amt_expected = files.len + directories.len + albums.len + string_bytes.len;
const amt_expected = files.len + directories.len + albums.len + string_bytes.len + waveforms.len;
if (amt_read != amt_expected) return error.UnexpectedFileSize;
 
try db.files.reIndexContext(gpa, .{});
 
shared/Db.zig added: 66, removed: 13, total 53
@@ -2,12 +2,18 @@ const std = @import("std");
const Db = @This();
const Allocator = std.mem.Allocator;
const Hash = std.hash.Wyhash;
const Twi = @import("Twi");
 
files: std.ArrayHashMapUnmanaged(File, void, File.Hasher, false),
directories: std.AutoArrayHashMapUnmanaged(Path, void),
albums: std.AutoArrayHashMapUnmanaged(Album, void),
string_bytes: std.ArrayListUnmanaged(u8),
queue_items: std.ArrayHashMapUnmanaged(QueueItem, void, QueueItem.Hasher, false),
/// Encoding:
/// * `max: [waveform_width / Twi.slice_len]u64`
/// * `min: [waveform_width / Twi.slice_len]u64`
/// * `rms: [waveform_width / Twi.slice_len]u64`
waveforms: std.ArrayListUnmanaged(u64),
state: State,
 
/// Used for finding the index inside `string_bytes`.
@@ -18,6 +24,8 @@ string_table: std.HashMapUnmanaged(
std.hash_map.default_max_load_percentage,
) = .{},
 
pub const waveform_width = 1920;
 
pub const empty: Db = .{
.files = .{},
.directories = .{},
@@ -25,6 +33,7 @@ pub const empty: Db = .{
.string_table = .{},
.albums = .{},
.queue_items = .{},
.waveforms = .{},
.state = .{
.flags = .{
.playing = false,
@@ -376,3 +385,22 @@ fn renderPath(db: *const Db, p: Path, buf: []u8) [:0]u8 {
buf[i] = 0;
return buf[0..i :0];
}
 
/// The waveform data position is determined based on File index alone.
pub fn resizeWaveformArray(db: *Db, gpa: Allocator, files_len: usize) Allocator.Error!void {
const channel_len = Db.waveform_width / Twi.slice_len; // number of u64 in the channel
const channel_count = @typeInfo(Twi.ColumnSlice).Struct.fields.len;
const waveform_size = channel_count * channel_len; // total number of u64 in the waveform
const end = waveform_size * files_len;
try db.waveforms.resize(gpa, end);
}
 
pub fn writeWaveformReserved(db: *Db, file_index: File.Index, twi: Twi) void {
const channel_len = Db.waveform_width / Twi.slice_len; // number of u64 in the channel
const channel_count = @typeInfo(Twi.ColumnSlice).Struct.fields.len;
const waveform_size = channel_count * channel_len; // total number of u64 in the waveform
const start = waveform_size * @intFromEnum(file_index);
@memcpy(db.waveforms.items[start + channel_len * 0 ..][0..channel_len], twi.slices.items(.max));
@memcpy(db.waveforms.items[start + channel_len * 1 ..][0..channel_len], twi.slices.items(.min));
@memcpy(db.waveforms.items[start + channel_len * 2 ..][0..channel_len], twi.slices.items(.rms));
}
 
tool/decode_twi.zig added: 66, removed: 13, total 53
@@ -1,6 +1,6 @@
const std = @import("std");
const json = std.json;
const Twi = @import("player").Twi;
const Twi = @import("Twi");
 
pub fn main() !void {
var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);