View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      https://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.codec.digest;
19  
20  import java.io.ByteArrayOutputStream;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.nio.charset.StandardCharsets;
24  import java.nio.file.DirectoryStream;
25  import java.nio.file.Files;
26  import java.nio.file.Path;
27  import java.security.MessageDigest;
28  import java.util.HashMap;
29  import java.util.Map;
30  import java.util.Objects;
31  import java.util.Set;
32  import java.util.TreeSet;
33  import java.util.function.Supplier;
34  
35  /**
36   * Computes <a href="https://git-scm.com/">Git</a> object identifiers and their generalizations described by the
37   * <a href="https://www.swhid.org/swhid-specification/">SWHID specification</a>.
38   *
39   * <p>
40   * When the hash algorithm is SHA-1, the identifiers produced by this class are identical to those used by Git.
41   * Other hash algorithms produce generalized identifiers as described by the SWHID specification.
42   * </p>
43   *
44   * <p>
45   * Git and SWHID treat file names and symbolic link targets as opaque byte sequences with no defined encoding. The
46   * identifiers produced here coincide with Git's or SWHID's own identifiers only if the original names and targets were
47   * UTF-8 encoded.
48   * </p>
49   *
50   * <p>
51   * This class is immutable and thread-safe. However, the {@link MessageDigest} instances passed to it generally won't be.
52   * </p>
53   *
54   * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
55   * @see <a href="https://www.swhid.org/swhid-specification/">SWHID Specification</a>
56   * @since 1.22.0
57   */
58  public class GitIdentifiers {
59  
60      /**
61       * Represents a single entry in a Git tree object.
62       *
63       * <p>
64       * A Git tree object encodes a directory snapshot. Each entry holds:
65       * </p>
66       * <ul>
67       *   <li>a {@link FileMode} that determines the Unix file mode (e.g. {@code 100644} for a regular file),</li>
68       *   <li>the entry name (file or directory name, without a path separator),</li>
69       *   <li>the raw object id of the referenced blob or sub-tree.</li>
70       * </ul>
71       *
72       * <p>
73       * Entries are ordered by {@link #compareTo} using Git's tree-sort rule: names are compared as unsigned UTF-8 bytes, and directory names are compared as if
74       * they ended with {@code '/'}, so that {@code foo/} sorts after {@code foobar}. Comparing the UTF-8 bytes rather than the Java {@link String} matches Git's
75       * order for names outside the Basic Multilingual Plane, whose UTF-16 code units do not sort in code point order.
76       * </p>
77       *
78       * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
79       * @see <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID Directory Identifier</a>
80       */
81      static class DirectoryEntry implements Comparable<DirectoryEntry> {
82  
83          private static String requireValidName(final String name) {
84              Objects.requireNonNull(name, "name");
85              if (name.isEmpty() || ".".equals(name) || "..".equals(name)) {
86                  throw new IllegalArgumentException("Entry name must not be empty, '.' or '..'");
87              }
88              if (name.indexOf('/') >= 0 || name.indexOf('\0') >= 0) {
89                  throw new IllegalArgumentException("Entry name must not contain '/' or NUL");
90              }
91              if (!StandardCharsets.UTF_8.newEncoder().canEncode(name)) {
92                  throw new IllegalArgumentException("Entry name must not contain unpaired surrogates");
93              }
94              return name;
95          }
96  
97          /**
98           * The entry name (file or directory name, no path separator).
99           */
100         private final String name;
101 
102         /**
103          * The raw object id of the referenced blob or sub-tree.
104          */
105         private final byte[] rawObjectId;
106 
107         /**
108          * The key used for ordering entries within a tree object, as the UTF-8 bytes Git itself compares.
109          *
110          * <p>
111          * Git appends {@code '/'} to directory names before comparing.
112          * </p>
113          */
114         private final byte[] sortKey;
115 
116         /**
117          * The Git object type, which determines the Unix file-mode prefix.
118          */
119         private final FileMode type;
120 
121         /**
122          * Constructs a new entry.
123          *
124          * @param name The nonempty entry name, not {@code .} or {@code ..}, without {@code '/'}, NUL or unpaired surrogates.
125          * @param type The type of the entry, not null.
126          * @param rawObjectId The id of the entry, not null.
127          */
128         DirectoryEntry(final String name, final FileMode type, final byte[] rawObjectId) {
129             this.name = requireValidName(name);
130             this.type = Objects.requireNonNull(type, "type");
131             this.sortKey = (type == FileMode.DIRECTORY ? name + "/" : name).getBytes(StandardCharsets.UTF_8);
132             this.rawObjectId = Objects.requireNonNull(rawObjectId, "rawObjectId");
133         }
134 
135         @Override
136         public int compareTo(final DirectoryEntry o) {
137             final byte[] a = sortKey;
138             final byte[] b = o.sortKey;
139             final int shared = Math.min(a.length, b.length);
140             for (int i = 0; i < shared; i++) {
141                 final int diff = (a[i] & 0xff) - (b[i] & 0xff);
142                 if (diff != 0) {
143                     return diff;
144                 }
145             }
146             return a.length != b.length ? a.length - b.length : name.compareTo(o.name);
147         }
148 
149         @Override
150         public boolean equals(final Object obj) {
151             if (obj == this) {
152                 return true;
153             }
154             if (!(obj instanceof DirectoryEntry)) {
155                 return false;
156             }
157             final DirectoryEntry other = (DirectoryEntry) obj;
158             return name.equals(other.name);
159         }
160 
161         @Override
162         public int hashCode() {
163             return name.hashCode();
164         }
165 
166     }
167 
168     /**
169      * The type of a Git tree entry, which maps to a Unix file-mode string.
170      *
171      * <p>
172      * Git encodes the file type and permission bits as an ASCII octal string that precedes the entry name in the binary tree format. The values defined here
173      * cover the four entry types that Git itself produces.
174      * </p>
175      *
176      * @see <a href="https://git-scm.com/book/en/v2/Git-Internals-Git-Objects">Git Internals – Git Objects</a>
177      */
178     public enum FileMode {
179 
180         /**
181          * A subdirectory. Subdirectories can only be specified by SHA or through a tree mark set with {@code --import-marks}.
182          *
183          * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
184          */
185         DIRECTORY(new byte[] { '4', '0', '0', '0', '0' }),
186 
187         /**
188          * A regular, but executable, file.
189          */
190         EXECUTABLE(new byte[] { '1', '0', '0', '7', '5', '5' }),
191 
192         /**
193          * A gitlink, SHA-1 of the object refers to a commit in another repository. Git links can only be specified either by SHA or through a commit mark. They
194          * are used to implement submodules.
195          *
196          * @see <a href="https://git-scm.com/docs/gitdatamodel">gitdatamodel - Git&#39;s core data model</a>
197          * @see <a href="https://git-scm.com/docs/git-fast-import">git-fast-import - Backend for fast Git data importers</a>
198          */
199         GIT_LINK(new byte[] { '1', '6', '0', '0', '0', '0' }),
200 
201         /**
202          * A regular (non-executable) file.
203          * <p>
204          * The majority of files in most projects use this mode. If in doubt, this is what you want.
205          * </p>
206          */
207         REGULAR(new byte[] { '1', '0', '0', '6', '4', '4' }),
208 
209         /**
210          * A symbolic link. The content of the file will be the link target.
211          */
212         SYMBOLIC_LINK(new byte[] { '1', '2', '0', '0', '0', '0' });
213 
214         private static FileMode get(final Path path) {
215             // Symbolic links first
216             if (Files.isSymbolicLink(path)) {
217                 return SYMBOLIC_LINK;
218             }
219             if (Files.isDirectory(path)) {
220                 return DIRECTORY;
221             }
222             if (Files.isExecutable(path)) {
223                 return EXECUTABLE;
224             }
225             return REGULAR;
226         }
227 
228         /**
229          * Serialized {@code mode}: since this is mutable, it must remain private.
230          */
231         private final byte[] modeBytes;
232 
233         FileMode(final byte[] modeBytes) {
234             // No need for a defensive copy since the array is private and never exposed,
235             this.modeBytes = modeBytes;
236         }
237     }
238 
239     /**
240      * Builds a Git tree identifier for a virtual directory structure, such as the contents of
241      * an archive.
242      */
243     public static final class TreeIdBuilder implements Supplier<byte[]> {
244 
245         /**
246          * Supplies a blob identifier that may throw {@link IOException}.
247          */
248         @FunctionalInterface
249         private interface BlobIdSupplier {
250             byte[] get() throws IOException;
251         }
252 
253         private final Map<String, TreeIdBuilder> dirEntries = new HashMap<>();
254         private final Map<String, DirectoryEntry> fileEntries = new HashMap<>();
255         private final MessageDigest messageDigest;
256 
257         private TreeIdBuilder(final MessageDigest messageDigest) {
258             this.messageDigest = Objects.requireNonNull(messageDigest, "messageDigest");
259         }
260 
261         /**
262          * Adds and returns the {@link TreeIdBuilder} for the named subdirectory, creating it if absent.
263          *
264          * @param name The relative path of the subdirectory in normalized form (may contain {@code '/'}).
265          * @return The {@link TreeIdBuilder} for the subdirectory.
266          * @throws IllegalArgumentException Thrown if any path component is {@code ".."}, contains NUL or contains unpaired surrogates.
267          */
268         public TreeIdBuilder addDirectory(final String name) {
269             TreeIdBuilder current = this;
270             for (final String component : name.split("/", -1)) {
271                 // Noop segments
272                 if (component.isEmpty() || ".".equals(component)) {
273                     continue;
274                 }
275                 current = current.dirEntries.computeIfAbsent(DirectoryEntry.requireValidName(component), k -> new TreeIdBuilder(messageDigest));
276             }
277             return current;
278         }
279 
280         private void addFile(final FileMode mode, final String name, final BlobIdSupplier blobId) throws IOException {
281             final int slash = name.lastIndexOf('/');
282             if (slash < 0) {
283                 DirectoryEntry.requireValidName(name);
284                 fileEntries.put(name, new DirectoryEntry(name, mode, blobId.get()));
285             } else {
286                 addDirectory(name.substring(0, slash)).addFile(mode, name.substring(slash + 1), blobId);
287             }
288         }
289 
290         /**
291          * Adds a file entry at the given path within this tree.
292          *
293          * <p>
294          * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
295          * </p>
296          *
297          * @param mode The file mode (e.g. {@link FileMode#REGULAR}).
298          * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
299          * @param data The file content.
300          * @throws IOException Thrown if an I/O error occurs.
301          * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
302          *                                  surrogates.
303          */
304         public void addFile(final FileMode mode, final String name, final byte[] data) throws IOException {
305             addFile(mode, name, () -> blobId(messageDigest, data));
306         }
307 
308         /**
309          * Adds a file entry at the given path within this tree, streaming content without buffering.
310          *
311          * <p>
312          * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
313          * </p>
314          *
315          * <p>
316          * The stream is eagerly drained.
317          * </p>
318          *
319          * @param mode     The file mode (e.g. {@link FileMode#REGULAR}).
320          * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
321          * @param dataSize The exact number of bytes in {@code data}.
322          * @param data     The file content.
323          * @throws IOException Thrown if the stream cannot be read, or does not contain exactly {@code dataSize} bytes.
324          * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
325          *                                  surrogates.
326          */
327         public void addFile(final FileMode mode, final String name, final long dataSize, final InputStream data) throws IOException {
328             addFile(mode, name, () -> blobId(messageDigest, dataSize, data));
329         }
330 
331         /**
332          * Adds a symbolic link entry at the given path within this tree.
333          *
334          * <p>
335          * If {@code name} contains {@code '/'}, intermediate subdirectories are created automatically.
336          * </p>
337          *
338          * @param name The relative path of the entry in normalized form(may contain {@code '/'}).
339          * @param target The target of the symbolic link.
340          * @throws IOException Thrown if an I/O error occurs.
341          * @throws IllegalArgumentException Thrown if the entry name is empty or {@code "."}, or any path component is {@code ".."}, contains NUL or contains unpaired
342          *                                  surrogates.
343          */
344         public void addSymbolicLink(final String name, final String target) throws IOException {
345             addFile(FileMode.SYMBOLIC_LINK, name, target.getBytes(StandardCharsets.UTF_8));
346         }
347 
348         /**
349          * Gets the Git tree identifier for this directory and all its descendants.
350          *
351          * @return The raw tree identifier bytes.
352          * @throws IllegalStateException Thrown if a file and a directory have the same name in this directory or any descendant.
353          */
354         @Override
355         public byte[] get() {
356             for (final String name : dirEntries.keySet()) {
357                 if (fileEntries.containsKey(name)) {
358                     throw new IllegalStateException("File and directory have the same name: " + name);
359                 }
360             }
361             final Set<DirectoryEntry> entries = new TreeSet<>(fileEntries.values());
362             dirEntries.forEach((k, v) -> entries.add(new DirectoryEntry(k, FileMode.DIRECTORY, v.get())));
363             final ByteArrayOutputStream baos = new ByteArrayOutputStream();
364             for (final DirectoryEntry entry : entries) {
365                 baos.write(entry.type.modeBytes, 0, entry.type.modeBytes.length);
366                 baos.write(' ');
367                 final byte[] bytes = entry.name.getBytes(StandardCharsets.UTF_8);
368                 baos.write(bytes, 0, bytes.length);
369                 baos.write('\0');
370                 baos.write(entry.rawObjectId, 0, entry.rawObjectId.length);
371             }
372             messageDigest.reset();
373             DigestUtils.updateDigest(messageDigest, getGitTreePrefix(baos.size()));
374             return DigestUtils.updateDigest(messageDigest, baos.toByteArray()).digest();
375         }
376 
377         private TreeIdBuilder populate(final Path directory) throws IOException {
378             try (DirectoryStream<Path> files = Files.newDirectoryStream(directory)) {
379                 for (final Path path : files) {
380                     final String name = Objects.toString(path.getFileName());
381                     final FileMode mode = FileMode.get(path);
382                     if (mode == FileMode.DIRECTORY) {
383                         addDirectory(name).populate(path);
384                     } else {
385                         addFile(mode, name, () -> blobId(messageDigest, path));
386                     }
387                 }
388             }
389             return this;
390         }
391     }
392 
393     /**
394      * Reads through a byte array and returns a generalized Git blob identifier.
395      *
396      * <p>
397      * The identifier is computed in the way described by the
398      * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
399      * algorithm.
400      * </p>
401      *
402      * <p>
403      * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
404      * </p>
405      *
406      * @param messageDigest The MessageDigest to use (for example SHA-1).
407      * @param data          Data to digest.
408      * @return A generalized Git blob identifier.
409      */
410     public static byte[] blobId(final MessageDigest messageDigest, final byte[] data) {
411         messageDigest.reset();
412         DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(data.length));
413         return DigestUtils.digest(messageDigest, data);
414     }
415 
416     /**
417      * Reads through a stream of known size and returns a generalized Git blob identifier, without buffering.
418      *
419      * <p>
420      * When the size of the content is known in advance, this overload streams {@code data} directly through
421      * the digest without buffering the full content in memory.
422      * </p>
423      *
424      * <p>
425      * The stream is drained to its end. If the number of bytes read differs from {@code dataSize}, an {@link IOException} is thrown.
426      * </p>
427      *
428      * <p>
429      * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
430      * </p>
431      *
432      * @param messageDigest The MessageDigest to use (for example SHA-1).
433      * @param dataSize      The exact number of bytes in {@code data}.
434      * @param data          Stream to digest.
435      * @return A generalized Git blob identifier.
436      * @throws IOException Thrown on error reading the stream, or if the stream does not contain exactly {@code dataSize} bytes.
437      */
438     public static byte[] blobId(final MessageDigest messageDigest, final long dataSize, final InputStream data) throws IOException {
439         messageDigest.reset();
440         DigestUtils.updateDigest(messageDigest, getGitBlobPrefix(dataSize));
441         final byte[] buffer = new byte[8192];
442         long actualSize = 0;
443         int read;
444         while ((read = data.read(buffer)) != -1) {
445             messageDigest.update(buffer, 0, read);
446             actualSize += read;
447         }
448         if (actualSize != dataSize) {
449             throw new IOException("Stream contained " + actualSize + " bytes, but dataSize declared " + dataSize + " bytes");
450         }
451         return messageDigest.digest();
452     }
453 
454     /**
455      * Reads through a file and returns a generalized Git blob identifier.
456      *
457      * <p>
458      * The identifier is computed in the way described by the
459      * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#52-contents">SWHID contents identifier</a>, but it can use any hash
460      * algorithm.
461      * </p>
462      *
463      * <p>
464      * When the hash algorithm is SHA-1, the identifier is identical to Git blob identifier and SWHID contents identifier.
465      * </p>
466      *
467      * @param messageDigest The MessageDigest to use (for example SHA-1).
468      * @param data          Path to the file to digest.
469      * @return A generalized Git blob identifier.
470      * @throws IOException Thrown on error accessing the file, or if the number of bytes read differs from its measured size.
471      */
472     public static byte[] blobId(final MessageDigest messageDigest, final Path data) throws IOException {
473         if (Files.isSymbolicLink(data)) {
474             final byte[] linkTarget = Files.readSymbolicLink(data).toString().getBytes(StandardCharsets.UTF_8);
475             return blobId(messageDigest, linkTarget);
476         }
477         final long dataSize = Files.size(data);
478         try (InputStream input = Files.newInputStream(data)) {
479             return blobId(messageDigest, dataSize, input);
480         }
481     }
482 
483     private static byte[] getGitBlobPrefix(final long dataSize) {
484         return getGitPrefix("blob", dataSize);
485     }
486 
487     private static byte[] getGitPrefix(final String type, final long dataSize) {
488         return (type + " " + dataSize + "\0").getBytes(StandardCharsets.UTF_8);
489     }
490 
491     private static byte[] getGitTreePrefix(final long dataSize) {
492         return getGitPrefix("tree", dataSize);
493     }
494 
495     /**
496      * Reads through a directory and returns a generalized Git tree identifier.
497      *
498      * <p>
499      * The identifier is computed in the way described by the
500      * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
501      * algorithm.
502      * </p>
503      *
504      * <p>
505      * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
506      * </p>
507      *
508      * @param messageDigest The MessageDigest to use (for example SHA-1).
509      * @param data          Path to the directory to digest.
510      * @return A generalized Git tree identifier.
511      * @throws IOException Thrown on error accessing the directory or its contents.
512      */
513     public static byte[] treeId(final MessageDigest messageDigest, final Path data) throws IOException {
514         return treeIdBuilder(messageDigest).populate(data).get();
515     }
516 
517     /**
518      * Returns a new {@link TreeIdBuilder} for constructing a generalized Git tree identifier from a virtual directory
519      * structure, such as the contents of an archive.
520      *
521      * <p>
522      * The identifier is computed in the way described by the
523      * <a href="https://www.swhid.org/swhid-specification/v1.2/5.Core_identifiers/#53-directories">SWHID directory identifier</a>, but it can use any hash
524      * algorithm.
525      * </p>
526      *
527      * <p>
528      * When the hash algorithm is SHA-1, the identifier is identical to Git tree identifier and SWHID directory identifier.
529      * </p>
530      *
531      * @param messageDigest The MessageDigest to use (for example SHA-1).
532      * @return A new {@link TreeIdBuilder}.
533      */
534     public static TreeIdBuilder treeIdBuilder(final MessageDigest messageDigest) {
535         return new TreeIdBuilder(messageDigest);
536     }
537 
538     private GitIdentifiers() {
539         // utility class
540     }
541 }