Skip to content

Commit 7435e03

Browse files
apocasapocas
authored
Merge pull request #824 from davidgamero/buildkit-progress
buildkit proto followProgress support
2 parents 5d4b860 + 3f9ec66 commit 7435e03

5 files changed

Lines changed: 622 additions & 0 deletions

File tree

‎examples/build/build_buildkit.js‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
var Docker = require('../../lib/docker'),
2+
tar = require('tar-fs');
3+
4+
var docker = new Docker({
5+
socketPath: '/var/run/docker.sock'
6+
});
7+
8+
var tarStream = tar.pack(process.cwd());
9+
10+
// BuildKit v2 builds return base64-encoded protobuf logs
11+
// Use docker.followProgress() to decode them automatically
12+
// (This works the same as docker.modem.followProgress but decodes BuildKit output)
13+
docker.buildImage(tarStream, {
14+
t: 'myimage:buildkit',
15+
version: '2' // Enable BuildKit
16+
}, function(error, stream) {
17+
if (error) {
18+
return console.error(error);
19+
}
20+
21+
// docker.followProgress works with both regular and BuildKit builds
22+
docker.followProgress(stream,
23+
function onFinished(err, result) {
24+
if (err) {
25+
console.error('Build failed:', err);
26+
} else {
27+
console.log('Build completed successfully!');
28+
}
29+
},
30+
function onProgress(event) {
31+
// Each event is already decoded and formatted
32+
if (event.stream) {
33+
process.stdout.write(event.stream);
34+
}
35+
}
36+
);
37+
});

‎lib/buildkit.js‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
var protobuf = require("protobufjs");
2+
var path = require("path");
3+
4+
// Constants
5+
var BUILDKIT_TRACE_ID = "moby.buildkit.trace";
6+
var BUILDKIT_IMAGE_ID = "moby.image.id";
7+
var PROTO_TYPE = "moby.buildkit.v1.StatusResponse";
8+
var ENCODING_UTF8 = "utf8";
9+
var ENCODING_BASE64 = "base64";
10+
11+
var StatusResponse;
12+
13+
// Load the protobuf schema
14+
function loadProto() {
15+
if (StatusResponse) return StatusResponse;
16+
17+
var root = protobuf.loadSync(
18+
path.resolve(__dirname, "proto", "buildkit_status.proto")
19+
);
20+
StatusResponse = root.lookupType(PROTO_TYPE);
21+
return StatusResponse;
22+
}
23+
24+
/**
25+
* Decodes a BuildKit trace message
26+
* @param {string} base64Data - Base64-encoded protobuf data from aux field
27+
* @returns {Object} Decoded status response with vertexes, logs, etc.
28+
*/
29+
function decodeBuildKitStatus(base64Data) {
30+
var StatusResponse = loadProto();
31+
32+
// Handle empty messages
33+
if (!base64Data || base64Data.length === 0) {
34+
return {
35+
vertexes: [],
36+
statuses: [],
37+
logs: [],
38+
warnings: []
39+
};
40+
}
41+
42+
var buffer = Buffer.from(base64Data, ENCODING_BASE64);
43+
var message = StatusResponse.decode(buffer);
44+
return StatusResponse.toObject(message, {
45+
longs: String,
46+
enums: String,
47+
bytes: String,
48+
defaults: true
49+
});
50+
}
51+
52+
/**
53+
* Formats BuildKit status into human-readable text
54+
* @param {Object} status - Decoded status response
55+
* @returns {string[]} Array of human-readable log lines
56+
*/
57+
function formatBuildKitStatus(status) {
58+
var lines = [];
59+
60+
// Process vertexes (build steps)
61+
if (status.vertexes && status.vertexes.length > 0) {
62+
status.vertexes.forEach(function(vertex) {
63+
if (vertex.name && vertex.started && !vertex.completed) {
64+
lines.push("[" + vertex.digest.substring(0, 12) + "] " + vertex.name);
65+
}
66+
if (vertex.error) {
67+
lines.push("ERROR: " + vertex.error);
68+
}
69+
if (vertex.completed && vertex.cached) {
70+
lines.push("CACHED: " + vertex.name);
71+
}
72+
});
73+
}
74+
75+
// Process logs (command output)
76+
if (status.logs && status.logs.length > 0) {
77+
status.logs.forEach(function(log) {
78+
var msg = Buffer.from(log.msg).toString(ENCODING_UTF8);
79+
if (msg.trim()) {
80+
lines.push(msg.trimEnd());
81+
}
82+
});
83+
}
84+
85+
// Process status updates (progress)
86+
if (status.statuses && status.statuses.length > 0) {
87+
status.statuses.forEach(function(s) {
88+
if (s.name && s.total > 0) {
89+
var percent = Math.floor((s.current / s.total) * 100);
90+
lines.push(s.name + ": " + percent + "% (" + s.current + "/" + s.total + ")");
91+
}
92+
});
93+
}
94+
95+
// Process warnings
96+
if (status.warnings && status.warnings.length > 0) {
97+
status.warnings.forEach(function(warning) {
98+
var msg = Buffer.from(warning.short).toString(ENCODING_UTF8);
99+
lines.push("WARNING: " + msg);
100+
});
101+
}
102+
103+
return lines;
104+
}
105+
106+
/**
107+
* Parse a BuildKit stream line and extract human-readable logs
108+
* @param {string} line - JSON line from build stream
109+
* @returns {Object} { isBuildKit: boolean, logs: string[], raw: Object }
110+
*/
111+
function parseBuildKitLine(line) {
112+
try {
113+
var json = JSON.parse(line);
114+
115+
// Check if it's a BuildKit trace message
116+
if (json.id === BUILDKIT_TRACE_ID && json.aux !== undefined) {
117+
var status = decodeBuildKitStatus(json.aux);
118+
var logs = formatBuildKitStatus(status);
119+
120+
return {
121+
isBuildKit: true,
122+
logs: logs,
123+
raw: status
124+
};
125+
}
126+
127+
// Check if it's the final image ID
128+
if (json.id === BUILDKIT_IMAGE_ID && json.aux && json.aux.ID) {
129+
return {
130+
isBuildKit: true,
131+
logs: ["Built image: " + json.aux.ID],
132+
raw: json.aux
133+
};
134+
}
135+
136+
// Not a BuildKit message
137+
return {
138+
isBuildKit: false,
139+
logs: [],
140+
raw: json
141+
};
142+
} catch (e) {
143+
return {
144+
isBuildKit: false,
145+
logs: [],
146+
raw: null,
147+
error: e.message
148+
};
149+
}
150+
}
151+
152+
/**
153+
* Follow progress of a stream, automatically handling both BuildKit and regular output.
154+
* This provides the same ergonomics as modem.followProgress but decodes BuildKit logs.
155+
*
156+
* @param {Stream} stream - Stream from buildImage(), pull(), push(), etc.
157+
* @param {Function} onFinished - Called when stream ends: (err, output) => void
158+
* @param {Function} onProgress - Called for each log event: (event) => void
159+
* @returns {void}
160+
*/
161+
function followProgress(stream, onFinished, onProgress) {
162+
var buffer = '';
163+
var output = [];
164+
var finished = false;
165+
166+
stream.on('data', onStreamEvent);
167+
stream.on('error', onStreamError);
168+
stream.on('end', onStreamEnd);
169+
stream.on('close', onStreamEnd);
170+
171+
function onStreamEvent(data) {
172+
buffer += data.toString();
173+
174+
// Process complete lines
175+
var lines = buffer.split('\n');
176+
buffer = lines.pop(); // Save incomplete line
177+
178+
lines.forEach(function(line) {
179+
if (!line.trim()) return;
180+
181+
processLine(line);
182+
});
183+
}
184+
185+
function processLine(line) {
186+
try {
187+
// Try to parse as BuildKit or regular Docker output
188+
var result = parseBuildKitLine(line);
189+
190+
if (result.isBuildKit) {
191+
// BuildKit message - create events from decoded logs
192+
result.logs.forEach(function(log) {
193+
var event = { stream: log + '\n' };
194+
output.push(event);
195+
if (onProgress) onProgress(event);
196+
});
197+
} else if (result.raw) {
198+
// Regular Docker message
199+
output.push(result.raw);
200+
if (onProgress) onProgress(result.raw);
201+
}
202+
} catch (e) {
203+
// If parsing fails, try plain JSON
204+
try {
205+
var json = JSON.parse(line);
206+
output.push(json);
207+
if (onProgress) onProgress(json);
208+
} catch (e2) {
209+
// Ignore parse errors
210+
}
211+
}
212+
}
213+
214+
function onStreamError(err) {
215+
finished = true;
216+
stream.removeListener('data', onStreamEvent);
217+
stream.removeListener('error', onStreamError);
218+
stream.removeListener('end', onStreamEnd);
219+
stream.removeListener('close', onStreamEnd);
220+
if (onFinished) onFinished(err, output);
221+
}
222+
223+
function onStreamEnd() {
224+
if (finished) return;
225+
finished = true;
226+
227+
// Process any remaining data in buffer
228+
if (buffer.trim()) {
229+
processLine(buffer);
230+
}
231+
232+
stream.removeListener('data', onStreamEvent);
233+
stream.removeListener('error', onStreamError);
234+
stream.removeListener('end', onStreamEnd);
235+
stream.removeListener('close', onStreamEnd);
236+
if (onFinished) onFinished(null, output);
237+
}
238+
}
239+
240+
module.exports = {
241+
followProgress: followProgress
242+
};

‎lib/docker.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,30 @@ Docker.prototype.buildImage = function(file, opts, callback) {
335335
}
336336
};
337337

338+
/**
339+
* Follow progress of a stream operation (build, pull, push, etc.) with automatic
340+
* BuildKit decoding.
341+
*
342+
* This method works identically to docker.modem.followProgress() but additionally
343+
* decodes BuildKit v2 build output. BuildKit emits base64-encoded protobuf messages
344+
* which this method transparently decodes into human-readable log events.
345+
*
346+
* Use this instead of docker.modem.followProgress() when:
347+
* - You're using BuildKit builds (version: "2")
348+
* - You want a single API that handles both regular and BuildKit output
349+
*
350+
* For non-BuildKit streams (pull, push, regular builds), behavior is identical
351+
* to docker.modem.followProgress().
352+
*
353+
* @param {Stream} stream - Stream from buildImage(), pull(), push(), etc.
354+
* @param {Function} onFinished - Called when stream ends: (err, output) => void
355+
* @param {Function} onProgress - Optional callback for each event: (event) => void
356+
*/
357+
Docker.prototype.followProgress = function(stream, onFinished, onProgress) {
358+
var buildkit = require('./buildkit');
359+
return buildkit.followProgress(stream, onFinished, onProgress);
360+
};
361+
338362
/**
339363
* Fetches a Container by ID
340364
* @param {String} id Container's ID

‎lib/proto/buildkit_status.proto‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
syntax = "proto3";
2+
3+
package moby.buildkit.v1;
4+
5+
// Minimal definitions for decoding BuildKit status messages
6+
// Based on https://github.com/moby/buildkit/blob/master/api/services/control/control.proto
7+
// Related to https://github.com/moby/buildkit/blob/master/solver/pb/ops.proto (vertices map to the solver op DAG)
8+
9+
message StatusResponse {
10+
repeated Vertex vertexes = 1;
11+
repeated VertexStatus statuses = 2;
12+
repeated VertexLog logs = 3;
13+
repeated VertexWarning warnings = 4;
14+
}
15+
16+
message Vertex {
17+
string digest = 1;
18+
repeated string inputs = 2;
19+
string name = 3;
20+
bool cached = 4;
21+
Timestamp started = 5;
22+
Timestamp completed = 6;
23+
string error = 7;
24+
ProgressGroup progressGroup = 8;
25+
}
26+
27+
message VertexStatus {
28+
string ID = 1;
29+
string vertex = 2;
30+
string name = 3;
31+
int64 current = 4;
32+
int64 total = 5;
33+
Timestamp timestamp = 6;
34+
Timestamp started = 7;
35+
Timestamp completed = 8;
36+
}
37+
38+
message VertexLog {
39+
string vertex = 1;
40+
Timestamp timestamp = 2;
41+
int64 stream = 3;
42+
bytes msg = 4;
43+
}
44+
45+
message VertexWarning {
46+
string vertex = 1;
47+
int64 level = 2;
48+
bytes short = 3;
49+
repeated bytes detail = 4;
50+
string url = 5;
51+
SourceInfo info = 6;
52+
repeated Range ranges = 7;
53+
}
54+
55+
message ProgressGroup {
56+
string id = 1;
57+
string name = 2;
58+
bool weak = 3;
59+
}
60+
61+
// Simplified Timestamp to match google.protobuf.Timestamp wire format
62+
message Timestamp {
63+
int64 seconds = 1;
64+
int32 nanos = 2;
65+
}
66+
67+
message SourceInfo {
68+
string filename = 1;
69+
bytes data = 2;
70+
// definition and language fields omitted - not needed for log decoding
71+
}
72+
73+
message Range {
74+
Position start = 1;
75+
Position end = 2;
76+
}
77+
78+
message Position {
79+
int32 line = 1;
80+
int32 character = 2;
81+
}

0 commit comments

Comments
 (0)