1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * https://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 package org.apache.commons.codec.digest;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.nio.charset.StandardCharsets;
24 import java.nio.file.DirectoryStream;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.security.MessageDigest;
28 import java.util.HashMap;
29 import java.util.Map;
30 import java.util.Objects;
31 import java.util.Set;
32 import java.util.TreeSet;
33 import java.util.function.Supplier;
34
35 /**
36 * Computes <a href="https://git-scm.com/">Git</a> object identifiers and their generalizations described by the
37 * <a href="https://www.swhid.org/swhid-specification/">SWHID specification</a>.
38 *
39 * <p>
40 * When the hash algorithm is SHA-1, the identifiers produced by this class are identical to those used by Git.
41 * Other hash algorithms produce generalized identifiers as described by the SWHID specification.
42 * </p>
43 *
44 * <p>
45 * Git and SWHID treat file names and symbolic link targets as opaque byte sequences with no defined encoding. The
46 * identifiers produced here coincide with Git's or SWHID's own identifiers only if the original names and targets were
47 * UTF-8 encoded.
48 * </p>
49 *
50 * <p>
51 * This class is immutable and thread-safe. However, the {@link MessageDigest} instances passed to it generally won't be.
52 * </p>
53 *
54 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
55 * @see <a href="https://www.swhid.org/swhid-specification/">SWHID Specification</a>
56 * @since 1.22.0
57 */
58 public class GitIdentifiers {
59
60 /**
61 * Represents a single entry in a Git tree object.
62 *
63 * <p>
64 * A Git tree object encodes a directory snapshot. Each entry holds:
65 * </p>
66 * <ul>
67 * <li>a {@link FileMode} that determines the Unix file mode (e.g. {@code 100644} for a regular file),</li>
68 * <li>the entry name (file or directory name, without a path separator),</li>
69 * <li>the raw object id of the referenced blob or sub-tree.</li>
70 * </ul>
71 *
72 * <p>
73 * Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if
74 * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} matches Git's
75 * order for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order.
76 * </p>
77 *
78 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
79 * @see <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID Directory Identifier</a>
80 */
81 static class DirectoryEntry implements Comparable<DirectoryEntry> {
82
83 private static String requireValidName(final String name) {
84 Objects.requireNonNull(name, "name");
85 if (name.isEmpty() || ".".equals(name) || "..".equals(name)) {
86 throw new IllegalArgumentException("Entry name must not be empty, '.' or '..'");
87 }
88 if (name.indexOf('/') >= 0 || name.indexOf('\0') >= 0) {
89 throw new IllegalArgumentException("Entry name must not contain '/' or NUL");
90 }
91 if (!StandardCharsets.UTF_8.newEncoder().canEncode(name)) {
92 throw new IllegalArgumentException("Entry name must not contain unpaired surrogates");
93 }
94 return name;
95 }
96
97 /**
98 * The entry name (file or directory name, no path separator).
99 */
100 private final String name;
101
102 /**
103 * The raw object id of the referenced blob or sub-tree.
104 */
105 private final byte[] rawObjectId;
106
107 /**
108 * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares.
109 *
110 * <p>
111 * Git appends {@code '/'} to directory names before comparing.
112 * </p>
113 */
114 private final byte[] sortKey;
115
116 /**
117 * The Git object type, which determines the Unix file-mode prefix.
118 */
119 private final FileMode type;
120
121 /**
122 * Constructs a new entry.
123 *
124 * @param name The nonempty entry name, not {@code .} or {@code ..}, without {@code '/'}, NUL or unpaired surrogates.
125 * @param type The type of the entry, not null.
126 * @param rawObjectId The id of the entry, not null.
127 */
128 DirectoryEntry(final String name, final FileMode type, final byte[] rawObjectId) {
129 this.name = requireValidName(name);
130 this.type = Objects.requireNonNull(type, "type");
131 this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8);
132 this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId");
133 }
134
135 @Override
136 public int compareTo(final DirectoryEntry o) {
137 final byte[] a = sortKey;
138 final byte[] b = o.sortKey;
139 final int shared = Math.min(a.length, b.length);
140 for (int i = 0; i < shared; i++) {
141 final int diff = (a[i] & 0xff) - (b[i] & 0xff);
142 if (diff != 0) {
143 return diff;
144 }
145 }
146 return a.length != b.length ? a.length - b.length : name.compareTo(o.name);
147 }
148
149 @Override
150 public boolean equals(final Object obj) {
151 if (obj == this) {
152 return true;
153 }
154 if (!(obj instanceof DirectoryEntry)) {
155 return false;
156 }
157 final DirectoryEntry other = (DirectoryEntry) obj;
158 return name.equals(other.name);
159 }
160
161 @Override
162 public int hashCode() {
163 return name.hashCode();
164 }
165
166 }
167
168 /**
169 * The type of a Git tree entry, which maps to a Unix file-mode string.
170 *
171 * <p>
172 * Git encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here
173 * cover the four entry types that Git itself produces.
174 * </p>
175 *
176 * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
177 */
178 public enum FileMode {
179
180 /**
181 * A subdirectory. Subdirectories can only be specified by SHA or through a tree mark set with {@code --import-marks}.
182 *
183 * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
184 */
185 DIRECTORY(new byte[] { '4', '0', '0', '0', '0' }),
186
187 /**
188 * A regular, but executable, file.
189 */
190 EXECUTABLE(new byte[] { '1', '0', '0', '7', '5', '5' }),
191
192 /**
193 * A gitlink, SHA-1 of the object refers to a commit in another repository. Git links can only be specified either by SHA or through a commit mark. They
194 * are used to implement submodules.
195 *
196 * @see <a href="https://git-scm.com/docs/gitdatamodel">gitdatamodel - Git's core data model</a>
197 * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
198 */
199 GIT_LINK(new byte[] { '1', '6', '0', '0', '0', '0' }),
200
201 /**
202 * A regular (non-executable) file.
203 * <p>
204 * The majority of files in most projects use this mode. If in doubt, this is what you want.
205 * </p>
206 */
207 REGULAR(new byte[] { '1', '0', '0', '6', '4', '4' }),
208
209 /**
210 * A symbolic link. The content of the file will be the link target.
211 */
212 SYMBOLIC_LINK(new byte[] { '1', '2', '0', '0', '0', '0' });
213
214 private static FileMode get(final Path path) {
215 // Symbolic links first
216 if (Files.isSymbolicLink(path)) {
217 return SYMBOLIC_LINK;
218 }
219 if (Files.isDirectory(path)) {
220 return DIRECTORY;
221 }
222 if (Files.isExecutable(path)) {
223 return EXECUTABLE;
224 }
225 return REGULAR;
226 }
227
228 /**
229 * Serialized {@code mode}: since this is mutable, it must remain private.
230 */
231 private final byte[] modeBytes;
232
233 FileMode(final byte[] modeBytes) {
234 // No need for a defensive copy since the array is private and never exposed,
235 this.modeBytes = modeBytes;
236 }
237 }
238
239 /**
240 * Builds a Git tree identifier for a virtual directory structure, such as the contents of
241 * an archive.
242 */
243 public static final class TreeIdBuilder implements Supplier<byte[]> {
244
245 /**
246 * Supplies a blob identifier that may throw {@link IOException}.
247 */
248 @FunctionalInterface
249 private interface BlobIdSupplier {
250 byte[] get() throws IOException;
251 }
252
253 private final Map<String, TreeIdBuilder> dirEntries = new HashMap<>();
254 private final Map<String, DirectoryEntry> fileEntries = new HashMap<>();
255 private final MessageDigest messageDigest;
256
257 private TreeIdBuilder(final MessageDigest messageDigest) {
258 this.messageDigest = Objects.requireNonNull(messageDigest, "messageDigest");
259 }
260
261 /**
262 * Adds and returns the {@link TreeIdBuilder} for the named subdirectory, creating it if absent.
263 *
264 * @param name The relative path of the subdirectory in normalized form (may contain {@code '/'}).
265 * @return The {@link TreeIdBuilder} for the subdirectory.
266 * @throws IllegalArgumentException Thrown if any path component is {@code ".."}, contains NUL or contains unpaired surrogates.
267 */
268 public TreeIdBuilder addDirectory(final String name) {
269 TreeIdBuilder current = this;
270 for (final String component : name.split("/", -1)) {
271 // Noop segments
272 if (component.isEmpty() || ".".equals(component)) {
273 continue;
274 }
275 current = current.dirEntries.computeIfAbsent(DirectoryEntry.requireValidName(component), k -> new TreeIdBuilder(messageDigest));
276 }
277 return current;
278 }
279
280 private void addFile(final FileMode mode, final String name, final BlobIdSupplier blobId) throws IOException {
281 final int slash = name.lastIndexOf('/');
282 if (slash < 0) {
283 DirectoryEntry.requireValidName(name);
284 fileEntries.put(name, new DirectoryEntry(name, mode, blobId.get()));
285 } else {
286 addDirectory(name.substring(0, slash)).addFile(mode, name.substring(slash + 1), blobId);
287 }
288 }
289
290 /**
291 * Adds a file entry at the given path within this tree.
292 *
293 * <p>
294 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
295 * </p>
296 *
297 * @param mode The file mode (e.g. {@link FileMode#REGULAR}).
298 * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
299 * @param data The file content.
300 * @throws IOException Thrown if an I/O error occurs.
301 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
302 * surrogates.
303 */
304 public void addFile(final FileMode mode, final String name, final byte[] data) throws IOException {
305 addFile(mode, name, () -> blobId(messageDigest, data));
306 }
307
308 /**
309 * Adds a file entry at the given path within this tree, streaming content without buffering.
310 *
311 * <p>
312 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
313 * </p>
314 *
315 * <p>
316 * The stream is eagerly drained.
317 * </p>
318 *
319 * @param mode The file mode (e.g. {@link FileMode#REGULAR}).
320 * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
321 * @param dataSize The exact number of bytes in {@code data}.
322 * @param data The file content.
323 * @throws IOException Thrown if the stream cannot be read, or does not contain exactly {@code dataSize} bytes.
324 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
325 * surrogates.
326 */
327 public void addFile(final FileMode mode, final String name, final long dataSize, final InputStream data) throws IOException {
328 addFile(mode, name, () -> blobId(messageDigest, dataSize, data));
329 }
330
331 /**
332 * Adds a symbolic link entry at the given path within this tree.
333 *
334 * <p>
335 * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
336 * </p>
337 *
338 * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
339 * @param target The target of the symbolic link.
340 * @throws IOException Thrown if an I/O error occurs.
341 * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
342 * surrogates.
343 */
344 public void addSymbolicLink(final String name, final String target) throws IOException {
345 addFile(FileMode.SYMBOLIC_LINK, name, target.getBytes(StandardCharsets.UTF_8));
346 }
347
348 /**
349 * Gets the Git tree identifier for this directory and all its descendants.
350 *
351 * @return The raw tree identifier bytes.
352 * @throws IllegalStateException Thrown if a file and a directory have the same name in this directory or any descendant.
353 */
354 @Override
355 public byte[] get() {
356 for (final String name : dirEntries.keySet()) {
357 if (fileEntries.containsKey(name)) {
358 throw new IllegalStateException("File and directory have the same name: " + name);
359 }
360 }
361 final Set<DirectoryEntry> entries = new TreeSet<>(fileEntries.values());
362 dirEntries.forEach((k, v) -> entries.add(new DirectoryEntry(k, FileMode.DIRECTORY, v.get())));
363 final ByteArrayOutputStream baos = new ByteArrayOutputStream();
364 for (final DirectoryEntry entry : entries) {
365 baos.write(entry.type.modeBytes, 0, entry.type.modeBytes.length);
366 baos.write(' ');
367 final byte[] bytes = entry.name.getBytes(StandardCharsets.UTF_8);
368 baos.write(bytes, 0, bytes.length);
369 baos.write('\0');
370 baos.write(entry.rawObjectId, 0, entry.rawObjectId.length);
371 }
372 messageDigest.reset();
373 DigestUtils.updateDigest(messageDigest, getGitTreePrefix(baos.size()));
374 return DigestUtils.updateDigest(messageDigest, baos.toByteArray()).digest();
375 }
376
377 private TreeIdBuilder populate(final Path directory) throws IOException {
378 try (DirectoryStream<Path> files = Files.newDirectoryStream(directory)) {
379 for (final Path path : files) {
380 final String name = Objects.toString(path.getFileName());
381 final FileMode mode = FileMode.get(path);
382 if (mode == FileMode.DIRECTORY) {
383 addDirectory(name).populate(path);
384 } else {
385 addFile(mode, name, () -> blobId(messageDigest, path));
386 }
387 }
388 }
389 return this;
390 }
391 }
392
393 /**
394 * Reads through a byte array and returns a generalized Git blob identifier.
395 *
396 * <p>
397 * The identifier is computed in the way described by the
398 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
399 * algorithm.
400 * </p>
401 *
402 * <p>
403 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
404 * </p>
405 *
406 * @param messageDigest The MessageDigest to use (for example SHA-1).
407 * @param data Data to digest.
408 * @return A generalized Git blob identifier.
409 */
410 public static byte[] blobId(final MessageDigest messageDigest, final byte[] data) {
411 messageDigest.reset();
412 DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(data.length));
413 return DigestUtils.digest(messageDigest, data);
414 }
415
416 /**
417 * Reads through a stream of known size and returns a generalized Git blob identifier, without buffering.
418 *
419 * <p>
420 * When the size of the content is known in advance, this overload streams {@code data} directly through
421 * the digest without buffering the full content in memory.
422 * </p>
423 *
424 * <p>
425 * The stream is drained to its end. If the number of bytes read differs from {@code dataSize}, an {@link IOException} is thrown.
426 * </p>
427 *
428 * <p>
429 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
430 * </p>
431 *
432 * @param messageDigest The MessageDigest to use (for example SHA-1).
433 * @param dataSize The exact number of bytes in {@code data}.
434 * @param data Stream to digest.
435 * @return A generalized Git blob identifier.
436 * @throws IOException Thrown on error reading the stream, or if the stream does not contain exactly {@code dataSize} bytes.
437 */
438 public static byte[] blobId(final MessageDigest messageDigest, final long dataSize, final InputStream data) throws IOException {
439 messageDigest.reset();
440 DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(dataSize));
441 final byte[] buffer = new byte[8192];
442 long actualSize = 0;
443 int read;
444 while ((read = data.read(buffer)) != -1) {
445 messageDigest.update(buffer, 0, read);
446 actualSize += read;
447 }
448 if (actualSize != dataSize) {
449 throw new IOException("Stream contained " + actualSize + " bytes, but dataSize declared " + dataSize + " bytes");
450 }
451 return messageDigest.digest();
452 }
453
454 /**
455 * Reads through a file and returns a generalized Git blob identifier.
456 *
457 * <p>
458 * The identifier is computed in the way described by the
459 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
460 * algorithm.
461 * </p>
462 *
463 * <p>
464 * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
465 * </p>
466 *
467 * @param messageDigest The MessageDigest to use (for example SHA-1).
468 * @param data Path to the file to digest.
469 * @return A generalized Git blob identifier.
470 * @throws IOException Thrown on error accessing the file, or if the number of bytes read differs from its measured size.
471 */
472 public static byte[] blobId(final MessageDigest messageDigest, final Path data) throws IOException {
473 if (Files.isSymbolicLink(data)) {
474 final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8);
475 return blobId(messageDigest, linkTarget);
476 }
477 final long dataSize = Files.size(data);
478 try (InputStream input = Files.newInputStream(data)) {
479 return blobId(messageDigest, dataSize, input);
480 }
481 }
482
483 private static byte[] getGitBlobPrefix(final long dataSize) {
484 return getGitPrefix("blob", dataSize);
485 }
486
487 private static byte[] getGitPrefix(final String type, final long dataSize) {
488 return (type + " " + dataSize + "\0").getBytes(StandardCharsets.UTF_8);
489 }
490
491 private static byte[] getGitTreePrefix(final long dataSize) {
492 return getGitPrefix("tree", dataSize);
493 }
494
495 /**
496 * Reads through a directory and returns a generalized Git tree identifier.
497 *
498 * <p>
499 * The identifier is computed in the way described by the
500 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
501 * algorithm.
502 * </p>
503 *
504 * <p>
505 * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
506 * </p>
507 *
508 * @param messageDigest The MessageDigest to use (for example SHA-1).
509 * @param data Path to the directory to digest.
510 * @return A generalized Git tree identifier.
511 * @throws IOException Thrown on error accessing the directory or its contents.
512 */
513 public static byte[] treeId(final MessageDigest messageDigest, final Path data) throws IOException {
514 return treeIdBuilder(messageDigest).populate(data).get();
515 }
516
517 /**
518 * Returns a new {@link TreeIdBuilder} for constructing a generalized Git tree identifier from a virtual directory
519 * structure, such as the contents of an archive.
520 *
521 * <p>
522 * The identifier is computed in the way described by the
523 * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
524 * algorithm.
525 * </p>
526 *
527 * <p>
528 * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
529 * </p>
530 *
531 * @param messageDigest The MessageDigest to use (for example SHA-1).
532 * @return A new {@link TreeIdBuilder}.
533 */
534 public static TreeIdBuilder treeIdBuilder(final MessageDigest messageDigest) {
535 return new TreeIdBuilder(messageDigest);
536 }
537
538 private GitIdentifiers() {
539 // utility class
540 }
541 }