Mercurial > repos > shellac > guppy_basecaller
annotate env/lib/python3.7/site-packages/bagit.py @ 0:26e78fe6e8c4 draft
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author | shellac |
---|---|
date | Sat, 02 May 2020 07:14:21 -0400 |
parents | |
children |
rev | line source |
---|---|
0
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1 #!/usr/bin/env python |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
2 # encoding: utf-8 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
3 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
4 from __future__ import absolute_import, division, print_function, unicode_literals |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
5 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
6 import argparse |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
7 import codecs |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
8 import gettext |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
9 import hashlib |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
10 import logging |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
11 import multiprocessing |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
12 import os |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
13 import re |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
14 import signal |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
15 import sys |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
16 import tempfile |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
17 import unicodedata |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
18 import warnings |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
19 from collections import defaultdict |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
20 from datetime import date |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
21 from functools import partial |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
22 from os.path import abspath, isdir, isfile, join |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
23 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
24 from pkg_resources import DistributionNotFound, get_distribution |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
25 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
26 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
27 from urllib.parse import urlparse |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
28 except ImportError: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
29 from urlparse import urlparse |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
30 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
31 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
32 def find_locale_dir(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
33 for prefix in (os.path.dirname(__file__), sys.prefix): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
34 locale_dir = os.path.join(prefix, "locale") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
35 if os.path.isdir(locale_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
36 return locale_dir |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
37 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
38 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
39 TRANSLATION_CATALOG = gettext.translation( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
40 "bagit-python", localedir=find_locale_dir(), fallback=True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
41 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
42 if sys.version_info < (3,): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
43 _ = TRANSLATION_CATALOG.ugettext |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
44 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
45 _ = TRANSLATION_CATALOG.gettext |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
46 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
47 MODULE_NAME = "bagit" if __name__ == "__main__" else __name__ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
48 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
49 LOGGER = logging.getLogger(MODULE_NAME) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
50 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
51 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
52 VERSION = get_distribution(MODULE_NAME).version |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
53 except DistributionNotFound: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
54 VERSION = "0.0.dev0" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
55 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
56 PROJECT_URL = "https://github.com/LibraryOfCongress/bagit-python" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
57 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
58 __doc__ = ( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
59 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
60 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
61 BagIt is a directory, filename convention for bundling an arbitrary set of |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
62 files with a manifest, checksums, and additional metadata. More about BagIt |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
63 can be found at: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
64 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
65 http://purl.org/net/bagit |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
66 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
67 bagit.py is a pure python drop in library and command line tool for creating, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
68 and working with BagIt directories. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
69 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
70 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
71 Command-Line Usage: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
72 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
73 Basic usage is to give bagit.py a directory to bag up: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
74 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
75 $ bagit.py my_directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
76 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
77 This does a bag-in-place operation where the current contents will be moved |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
78 into the appropriate BagIt structure and the metadata files will be created. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
79 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
80 You can bag multiple directories if you wish: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
81 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
82 $ bagit.py directory1 directory2 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
83 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
84 Optionally you can provide metadata which will be stored in bag-info.txt: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
85 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
86 $ bagit.py --source-organization "Library of Congress" directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
87 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
88 You can also select which manifest algorithms will be used: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
89 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
90 $ bagit.py --sha1 --md5 --sha256 --sha512 directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
91 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
92 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
93 Using BagIt from your Python code: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
94 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
95 import bagit |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
96 bag = bagit.make_bag('example-directory', {'Contact-Name': 'Ed Summers'}) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
97 print(bag.entries) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
98 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
99 For more information or to contribute to bagit-python's development, please |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
100 visit %(PROJECT_URL)s |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
101 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
102 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
103 % globals() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
104 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
105 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
106 # standard bag-info.txt metadata |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
107 STANDARD_BAG_INFO_HEADERS = [ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
108 "Source-Organization", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
109 "Organization-Address", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
110 "Contact-Name", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
111 "Contact-Phone", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
112 "Contact-Email", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
113 "External-Description", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
114 "External-Identifier", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
115 "Bag-Size", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
116 "Bag-Group-Identifier", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
117 "Bag-Count", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
118 "Internal-Sender-Identifier", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
119 "Internal-Sender-Description", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
120 "BagIt-Profile-Identifier", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
121 # Bagging-Date is autogenerated |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
122 # Payload-Oxum is autogenerated |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
123 ] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
124 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
125 CHECKSUM_ALGOS = hashlib.algorithms_guaranteed |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
126 DEFAULT_CHECKSUMS = ["sha256", "sha512"] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
127 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
128 #: Block size used when reading files for hashing: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
129 HASH_BLOCK_SIZE = 512 * 1024 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
130 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
131 #: Convenience function used everywhere we want to open a file to read text |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
132 #: rather than undecoded bytes: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
133 open_text_file = partial(codecs.open, encoding="utf-8", errors="strict") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
134 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
135 # This is the same as decoding the byte values in codecs.BOM: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
136 UNICODE_BYTE_ORDER_MARK = "\uFEFF" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
137 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
138 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
139 def make_bag( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
140 bag_dir, bag_info=None, processes=1, checksums=None, checksum=None, encoding="utf-8" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
141 ): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
142 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
143 Convert a given directory into a bag. You can pass in arbitrary |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
144 key/value pairs to put into the bag-info.txt metadata file as |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
145 the bag_info dictionary. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
146 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
147 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
148 if checksum is not None: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
149 warnings.warn( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
150 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
151 "The `checksum` argument for `make_bag` should be replaced with `checksums`" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
152 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
153 DeprecationWarning, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
154 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
155 checksums = checksum |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
156 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
157 if checksums is None: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
158 checksums = DEFAULT_CHECKSUMS |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
159 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
160 bag_dir = os.path.abspath(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
161 cwd = os.path.abspath(os.path.curdir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
162 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
163 if cwd.startswith(bag_dir) and cwd != bag_dir: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
164 raise RuntimeError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
165 _("Bagging a parent of the current directory is not supported") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
166 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
167 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
168 LOGGER.info(_("Creating bag for directory %s"), bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
169 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
170 if not os.path.isdir(bag_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
171 LOGGER.error(_("Bag directory %s does not exist"), bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
172 raise RuntimeError(_("Bag directory %s does not exist") % bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
173 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
174 # FIXME: we should do the permissions checks before changing directories |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
175 old_dir = os.path.abspath(os.path.curdir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
176 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
177 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
178 # TODO: These two checks are currently redundant since an unreadable directory will also |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
179 # often be unwritable, and this code will require review when we add the option to |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
180 # bag to a destination other than the source. It would be nice if we could avoid |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
181 # walking the directory tree more than once even if most filesystems will cache it |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
182 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
183 unbaggable = _can_bag(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
184 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
185 if unbaggable: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
186 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
187 _("Unable to write to the following directories and files:\n%s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
188 unbaggable, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
189 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
190 raise BagError(_("Missing permissions to move all files and directories")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
191 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
192 unreadable_dirs, unreadable_files = _can_read(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
193 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
194 if unreadable_dirs or unreadable_files: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
195 if unreadable_dirs: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
196 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
197 _("The following directories do not have read permissions:\n%s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
198 unreadable_dirs, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
199 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
200 if unreadable_files: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
201 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
202 _("The following files do not have read permissions:\n%s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
203 unreadable_files, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
204 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
205 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
206 _("Read permissions are required to calculate file fixities") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
207 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
208 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
209 LOGGER.info(_("Creating data directory")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
210 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
211 # FIXME: if we calculate full paths we won't need to deal with changing directories |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
212 os.chdir(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
213 cwd = os.getcwd() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
214 temp_data = tempfile.mkdtemp(dir=cwd) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
215 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
216 for f in os.listdir("."): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
217 if os.path.abspath(f) == temp_data: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
218 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
219 new_f = os.path.join(temp_data, f) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
220 LOGGER.info( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
221 _("Moving %(source)s to %(destination)s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
222 {"source": f, "destination": new_f}, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
223 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
224 os.rename(f, new_f) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
225 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
226 LOGGER.info( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
227 _("Moving %(source)s to %(destination)s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
228 {"source": temp_data, "destination": "data"}, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
229 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
230 os.rename(temp_data, "data") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
231 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
232 # permissions for the payload directory should match those of the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
233 # original directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
234 os.chmod("data", os.stat(cwd).st_mode) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
235 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
236 total_bytes, total_files = make_manifests( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
237 "data", processes, algorithms=checksums, encoding=encoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
238 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
239 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
240 LOGGER.info(_("Creating bagit.txt")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
241 txt = """BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
242 with open_text_file("bagit.txt", "w") as bagit_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
243 bagit_file.write(txt) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
244 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
245 LOGGER.info(_("Creating bag-info.txt")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
246 if bag_info is None: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
247 bag_info = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
248 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
249 # allow 'Bagging-Date' and 'Bag-Software-Agent' to be overidden |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
250 if "Bagging-Date" not in bag_info: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
251 bag_info["Bagging-Date"] = date.strftime(date.today(), "%Y-%m-%d") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
252 if "Bag-Software-Agent" not in bag_info: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
253 bag_info["Bag-Software-Agent"] = "bagit.py v%s <%s>" % ( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
254 VERSION, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
255 PROJECT_URL, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
256 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
257 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
258 bag_info["Payload-Oxum"] = "%s.%s" % (total_bytes, total_files) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
259 _make_tag_file("bag-info.txt", bag_info) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
260 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
261 for c in checksums: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
262 _make_tagmanifest_file(c, bag_dir, encoding="utf-8") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
263 except Exception: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
264 LOGGER.exception(_("An error occurred creating a bag in %s"), bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
265 raise |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
266 finally: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
267 os.chdir(old_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
268 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
269 return Bag(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
270 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
271 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
272 class Bag(object): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
273 """A representation of a bag.""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
274 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
275 valid_files = ["bagit.txt", "fetch.txt"] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
276 valid_directories = ["data"] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
277 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
278 def __init__(self, path=None): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
279 super(Bag, self).__init__() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
280 self.tags = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
281 self.info = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
282 #: Dictionary of manifest entries and the checksum values for each |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
283 #: algorithm: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
284 self.entries = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
285 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
286 # To reliably handle Unicode normalization differences, we maintain |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
287 # lookup dictionaries in both directions for the filenames read from |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
288 # the filesystem and the manifests so we can handle cases where the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
289 # normalization form changed between the bag being created and read. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
290 # See https://github.com/LibraryOfCongress/bagit-python/issues/51. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
291 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
292 #: maps Unicode-normalized values to the raw value from the filesystem |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
293 self.normalized_filesystem_names = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
294 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
295 #: maps Unicode-normalized values to the raw value in the manifest |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
296 self.normalized_manifest_names = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
297 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
298 self.algorithms = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
299 self.tag_file_name = None |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
300 self.path = abspath(path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
301 if path: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
302 # if path ends in a path separator, strip it off |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
303 if path[-1] == os.sep: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
304 self.path = path[:-1] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
305 self._open() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
306 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
307 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
308 # FIXME: develop a more informative string representation for a Bag |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
309 return self.path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
310 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
311 @property |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
312 def algs(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
313 warnings.warn(_("Use Bag.algorithms instead of Bag.algs"), DeprecationWarning) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
314 return self.algorithms |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
315 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
316 @property |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
317 def version(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
318 warnings.warn( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
319 _("Use the Bag.version_info tuple instead of Bag.version"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
320 DeprecationWarning, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
321 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
322 return self._version |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
323 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
324 def _open(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
325 # Open the bagit.txt file, and load any tags from it, including |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
326 # the required version and encoding. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
327 bagit_file_path = os.path.join(self.path, "bagit.txt") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
328 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
329 if not isfile(bagit_file_path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
330 raise BagError(_("Expected bagit.txt does not exist: %s") % bagit_file_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
331 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
332 self.tags = tags = _load_tag_file(bagit_file_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
333 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
334 required_tags = ("BagIt-Version", "Tag-File-Character-Encoding") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
335 missing_tags = [i for i in required_tags if i not in tags] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
336 if missing_tags: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
337 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
338 _("Missing required tag in bagit.txt: %s") % ", ".join(missing_tags) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
339 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
340 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
341 # To avoid breaking existing code we'll leave self.version as the string |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
342 # and parse it into a numeric version_info tuple. In version 2.0 we can |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
343 # break that. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
344 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
345 self._version = tags["BagIt-Version"] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
346 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
347 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
348 self.version_info = tuple(int(i) for i in self._version.split(".", 1)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
349 except ValueError: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
350 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
351 _("Bag version numbers must be MAJOR.MINOR numbers, not %s") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
352 % self._version |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
353 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
354 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
355 if (0, 93) <= self.version_info <= (0, 95): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
356 self.tag_file_name = "package-info.txt" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
357 elif (0, 96) <= self.version_info < (2,): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
358 self.tag_file_name = "bag-info.txt" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
359 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
360 raise BagError(_("Unsupported bag version: %s") % self._version) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
361 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
362 self.encoding = tags["Tag-File-Character-Encoding"] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
363 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
364 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
365 codecs.lookup(self.encoding) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
366 except LookupError: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
367 raise BagValidationError(_("Unsupported encoding: %s") % self.encoding) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
368 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
369 info_file_path = os.path.join(self.path, self.tag_file_name) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
370 if os.path.exists(info_file_path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
371 self.info = _load_tag_file(info_file_path, encoding=self.encoding) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
372 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
373 self._load_manifests() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
374 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
375 def manifest_files(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
376 for filename in ["manifest-%s.txt" % a for a in CHECKSUM_ALGOS]: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
377 f = os.path.join(self.path, filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
378 if isfile(f): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
379 yield f |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
380 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
381 def tagmanifest_files(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
382 for filename in ["tagmanifest-%s.txt" % a for a in CHECKSUM_ALGOS]: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
383 f = os.path.join(self.path, filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
384 if isfile(f): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
385 yield f |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
386 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
387 def compare_manifests_with_fs(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
388 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
389 Compare the filenames in the manifests to the filenames present on the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
390 local filesystem and returns two lists of the files which are only |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
391 present in the manifests and the files which are only present on the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
392 local filesystem, respectively. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
393 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
394 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
395 # We compare the filenames after Unicode normalization so we can |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
396 # reliably detect normalization changes after bag creation: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
397 files_on_fs = set(normalize_unicode(i) for i in self.payload_files()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
398 files_in_manifest = set( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
399 normalize_unicode(i) for i in self.payload_entries().keys() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
400 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
401 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
402 if self.version_info >= (0, 97): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
403 files_in_manifest.update(self.missing_optional_tagfiles()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
404 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
405 only_on_fs = list() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
406 only_in_manifest = list() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
407 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
408 for i in files_on_fs.difference(files_in_manifest): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
409 only_on_fs.append(self.normalized_filesystem_names[i]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
410 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
411 for i in files_in_manifest.difference(files_on_fs): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
412 only_in_manifest.append(self.normalized_manifest_names[i]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
413 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
414 return only_in_manifest, only_on_fs |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
415 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
416 def compare_fetch_with_fs(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
417 """Compares the fetch entries with the files actually |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
418 in the payload, and returns a list of all the files |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
419 that still need to be fetched. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
420 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
421 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
422 files_on_fs = set(self.payload_files()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
423 files_in_fetch = set(self.files_to_be_fetched()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
424 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
425 return list(files_in_fetch - files_on_fs) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
426 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
427 def payload_files(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
428 """Returns a list of filenames which are present on the local filesystem""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
429 payload_dir = os.path.join(self.path, "data") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
430 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
431 for dirpath, _, filenames in os.walk(payload_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
432 for f in filenames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
433 # Jump through some hoops here to make the payload files are |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
434 # returned with the directory structure relative to the base |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
435 # directory rather than the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
436 normalized_f = os.path.normpath(f) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
437 rel_path = os.path.relpath( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
438 os.path.join(dirpath, normalized_f), start=self.path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
439 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
440 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
441 self.normalized_filesystem_names[normalize_unicode(rel_path)] = rel_path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
442 yield rel_path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
443 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
444 def payload_entries(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
445 """Return a dictionary of items """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
446 # Don't use dict comprehension (compatibility with Python < 2.7) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
447 return dict( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
448 (key, value) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
449 for (key, value) in self.entries.items() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
450 if key.startswith("data" + os.sep) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
451 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
452 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
453 def save(self, processes=1, manifests=False): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
454 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
455 save will persist any changes that have been made to the bag |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
456 metadata (self.info). |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
457 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
458 If you have modified the payload of the bag (added, modified, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
459 removed files in the data directory) and want to regenerate manifests |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
460 set the manifests parameter to True. The default is False since you |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
461 wouldn't want a save to accidentally create a new manifest for |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
462 a corrupted bag. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
463 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
464 If you want to control the number of processes that are used when |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
465 recalculating checksums use the processes parameter. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
466 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
467 # Error checking |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
468 if not self.path: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
469 raise BagError(_("Bag.save() called before setting the path!")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
470 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
471 if not os.access(self.path, os.R_OK | os.W_OK | os.X_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
472 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
473 _("Cannot save bag to non-existent or inaccessible directory %s") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
474 % self.path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
475 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
476 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
477 unbaggable = _can_bag(self.path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
478 if unbaggable: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
479 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
480 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
481 "Missing write permissions for the following directories and files:\n%s" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
482 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
483 unbaggable, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
484 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
485 raise BagError(_("Missing permissions to move all files and directories")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
486 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
487 unreadable_dirs, unreadable_files = _can_read(self.path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
488 if unreadable_dirs or unreadable_files: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
489 if unreadable_dirs: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
490 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
491 _("The following directories do not have read permissions:\n%s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
492 unreadable_dirs, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
493 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
494 if unreadable_files: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
495 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
496 _("The following files do not have read permissions:\n%s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
497 unreadable_files, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
498 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
499 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
500 _("Read permissions are required to calculate file fixities") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
501 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
502 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
503 # Change working directory to bag directory so helper functions work |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
504 old_dir = os.path.abspath(os.path.curdir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
505 os.chdir(self.path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
506 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
507 # Generate new manifest files |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
508 if manifests: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
509 total_bytes, total_files = make_manifests( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
510 "data", processes, algorithms=self.algorithms, encoding=self.encoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
511 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
512 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
513 # Update Payload-Oxum |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
514 LOGGER.info(_("Updating Payload-Oxum in %s"), self.tag_file_name) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
515 self.info["Payload-Oxum"] = "%s.%s" % (total_bytes, total_files) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
516 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
517 _make_tag_file(self.tag_file_name, self.info) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
518 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
519 # Update tag-manifest for changes to manifest & bag-info files |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
520 for alg in self.algorithms: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
521 _make_tagmanifest_file(alg, self.path, encoding=self.encoding) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
522 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
523 # Reload the manifests |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
524 self._load_manifests() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
525 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
526 os.chdir(old_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
527 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
528 def tagfile_entries(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
529 return dict( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
530 (key, value) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
531 for (key, value) in self.entries.items() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
532 if not key.startswith("data" + os.sep) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
533 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
534 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
535 def missing_optional_tagfiles(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
536 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
537 From v0.97 we need to validate any tagfiles listed |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
538 in the optional tagmanifest(s). As there is no mandatory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
539 directory structure for additional tagfiles we can |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
540 only check for entries with missing files (not missing |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
541 entries for existing files). |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
542 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
543 for tagfilepath in self.tagfile_entries().keys(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
544 if not os.path.isfile(os.path.join(self.path, tagfilepath)): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
545 yield tagfilepath |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
546 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
547 def fetch_entries(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
548 """Load fetch.txt if present and iterate over its contents |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
549 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
550 yields (url, size, filename) tuples |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
551 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
552 raises BagError for errors such as an unsafe filename referencing |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
553 data outside of the bag directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
554 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
555 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
556 fetch_file_path = os.path.join(self.path, "fetch.txt") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
557 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
558 if isfile(fetch_file_path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
559 with open_text_file( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
560 fetch_file_path, "r", encoding=self.encoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
561 ) as fetch_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
562 for line in fetch_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
563 url, file_size, filename = line.strip().split(None, 2) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
564 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
565 if self._path_is_dangerous(filename): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
566 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
567 _('Path "%(payload_file)s" in "%(source_file)s" is unsafe') |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
568 % { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
569 "payload_file": filename, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
570 "source_file": os.path.join(self.path, "fetch.txt"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
571 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
572 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
573 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
574 yield url, file_size, filename |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
575 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
576 def files_to_be_fetched(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
577 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
578 Convenience wrapper for fetch_entries which returns only the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
579 local filename |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
580 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
581 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
582 for url, file_size, filename in self.fetch_entries(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
583 yield filename |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
584 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
585 def has_oxum(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
586 return "Payload-Oxum" in self.info |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
587 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
588 def validate(self, processes=1, fast=False, completeness_only=False): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
589 """Checks the structure and contents are valid. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
590 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
591 If you supply the parameter fast=True the Payload-Oxum (if present) will |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
592 be used to check that the payload files are present and accounted for, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
593 instead of re-calculating fixities and comparing them against the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
594 manifest. By default validate() will re-calculate fixities (fast=False). |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
595 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
596 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
597 self._validate_structure() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
598 self._validate_bagittxt() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
599 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
600 self.validate_fetch() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
601 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
602 self._validate_contents( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
603 processes=processes, fast=fast, completeness_only=completeness_only |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
604 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
605 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
606 return True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
607 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
608 def is_valid(self, fast=False, completeness_only=False): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
609 """Returns validation success or failure as boolean. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
610 Optional fast parameter passed directly to validate(). |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
611 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
612 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
613 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
614 self.validate(fast=fast, completeness_only=completeness_only) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
615 except BagError: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
616 return False |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
617 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
618 return True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
619 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
620 def _load_manifests(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
621 self.entries = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
622 manifests = list(self.manifest_files()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
623 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
624 if self.version_info >= (0, 97): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
625 # v0.97+ requires that optional tagfiles are verified. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
626 manifests += list(self.tagmanifest_files()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
627 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
628 for manifest_filename in manifests: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
629 if not manifest_filename.find("tagmanifest-") is -1: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
630 search = "tagmanifest-" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
631 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
632 search = "manifest-" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
633 alg = ( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
634 os.path.basename(manifest_filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
635 .replace(search, "") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
636 .replace(".txt", "") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
637 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
638 if alg not in self.algorithms: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
639 self.algorithms.append(alg) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
640 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
641 with open_text_file( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
642 manifest_filename, "r", encoding=self.encoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
643 ) as manifest_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
644 if manifest_file.encoding.startswith("UTF"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
645 # We'll check the first character to see if it's a BOM: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
646 if manifest_file.read(1) == UNICODE_BYTE_ORDER_MARK: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
647 # We'll skip it either way by letting line decoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
648 # happen at the new offset but we will issue a warning |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
649 # for UTF-8 since the presence of a BOM is contrary to |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
650 # the BagIt specification: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
651 if manifest_file.encoding == "UTF-8": |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
652 LOGGER.warning( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
653 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
654 "%s is encoded using UTF-8 but contains an unnecessary" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
655 " byte-order mark, which is not in compliance with the" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
656 " BagIt RFC" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
657 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
658 manifest_file.name, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
659 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
660 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
661 manifest_file.seek(0) # Pretend the first read never happened |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
662 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
663 for line in manifest_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
664 line = line.strip() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
665 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
666 # Ignore blank lines and comments. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
667 if line == "" or line.startswith("#"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
668 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
669 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
670 entry = line.split(None, 1) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
671 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
672 # Format is FILENAME *CHECKSUM |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
673 if len(entry) != 2: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
674 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
675 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
676 "%(bag)s: Invalid %(algorithm)s manifest entry: %(line)s" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
677 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
678 {"bag": self, "algorithm": alg, "line": line}, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
679 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
680 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
681 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
682 entry_hash = entry[0] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
683 entry_path = os.path.normpath(entry[1].lstrip("*")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
684 entry_path = _decode_filename(entry_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
685 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
686 if self._path_is_dangerous(entry_path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
687 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
688 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
689 'Path "%(payload_file)s" in manifest "%(manifest_file)s" is unsafe' |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
690 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
691 % { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
692 "payload_file": entry_path, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
693 "manifest_file": manifest_file.name, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
694 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
695 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
696 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
697 entry_hashes = self.entries.setdefault(entry_path, {}) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
698 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
699 if alg in entry_hashes: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
700 warning_ctx = { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
701 "bag": self, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
702 "algorithm": alg, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
703 "filename": entry_path, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
704 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
705 if entry_hashes[alg] == entry_hash: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
706 msg = _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
707 "%(bag)s: %(algorithm)s manifest lists %(filename)s" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
708 " multiple times with the same value" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
709 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
710 if self.version_info >= (1,): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
711 raise BagError(msg % warning_ctx) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
712 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
713 LOGGER.warning(msg, warning_ctx) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
714 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
715 raise BagError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
716 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
717 "%(bag)s: %(algorithm)s manifest lists %(filename)s" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
718 " multiple times with conflicting values" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
719 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
720 % warning_ctx |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
721 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
722 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
723 entry_hashes[alg] = entry_hash |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
724 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
725 self.normalized_manifest_names.update( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
726 (normalize_unicode(i), i) for i in self.entries.keys() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
727 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
728 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
729 def _validate_structure(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
730 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
731 Checks the structure of the bag to determine whether it conforms to the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
732 BagIt spec. Returns true on success, otherwise it will raise a |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
733 BagValidationError exception. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
734 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
735 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
736 self._validate_structure_payload_directory() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
737 self._validate_structure_tag_files() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
738 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
739 def _validate_structure_payload_directory(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
740 data_dir_path = os.path.join(self.path, "data") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
741 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
742 if not isdir(data_dir_path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
743 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
744 _("Expected data directory %s does not exist") % data_dir_path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
745 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
746 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
747 def _validate_structure_tag_files(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
748 # Note: we deviate somewhat from v0.96 of the spec in that it allows |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
749 # other files and directories to be present in the base directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
750 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
751 if not list(self.manifest_files()): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
752 raise BagValidationError(_("No manifest files found")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
753 if "bagit.txt" not in os.listdir(self.path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
754 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
755 _('Expected %s to contain "bagit.txt"') % self.path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
756 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
757 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
758 def validate_fetch(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
759 """Validate the fetch.txt file |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
760 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
761 Raises `BagError` for errors and otherwise returns no value |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
762 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
763 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
764 for url, file_size, filename in self.fetch_entries(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
765 # fetch_entries will raise a BagError for unsafe filenames |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
766 # so at this point we will check only that the URL is minimally |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
767 # well formed: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
768 parsed_url = urlparse(url) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
769 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
770 if not all((parsed_url.scheme, parsed_url.netloc)): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
771 raise BagError(_("Malformed URL in fetch.txt: %s") % url) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
772 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
773 def _validate_contents(self, processes=1, fast=False, completeness_only=False): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
774 if fast and not self.has_oxum(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
775 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
776 _("Fast validation requires bag-info.txt to include Payload-Oxum") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
777 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
778 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
779 # Perform the fast file count + size check so we can fail early: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
780 self._validate_oxum() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
781 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
782 if fast: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
783 return |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
784 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
785 self._validate_completeness() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
786 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
787 if completeness_only: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
788 return |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
789 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
790 self._validate_entries(processes) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
791 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
792 def _validate_oxum(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
793 oxum = self.info.get("Payload-Oxum") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
794 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
795 if oxum is None: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
796 return |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
797 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
798 # If multiple Payload-Oxum tags (bad idea) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
799 # use the first listed in bag-info.txt |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
800 if isinstance(oxum, list): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
801 LOGGER.warning(_("bag-info.txt defines multiple Payload-Oxum values!")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
802 oxum = oxum[0] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
803 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
804 oxum_byte_count, oxum_file_count = oxum.split(".", 1) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
805 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
806 if not oxum_byte_count.isdigit() or not oxum_file_count.isdigit(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
807 raise BagError(_("Malformed Payload-Oxum value: %s") % oxum) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
808 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
809 oxum_byte_count = int(oxum_byte_count) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
810 oxum_file_count = int(oxum_file_count) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
811 total_bytes = 0 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
812 total_files = 0 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
813 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
814 for payload_file in self.payload_files(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
815 payload_file = os.path.join(self.path, payload_file) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
816 total_bytes += os.stat(payload_file).st_size |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
817 total_files += 1 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
818 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
819 if oxum_file_count != total_files or oxum_byte_count != total_bytes: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
820 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
821 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
822 "Payload-Oxum validation failed." |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
823 " Expected %(oxum_file_count)d files and %(oxum_byte_count)d bytes" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
824 " but found %(found_file_count)d files and %(found_byte_count)d bytes" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
825 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
826 % { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
827 "found_file_count": total_files, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
828 "found_byte_count": total_bytes, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
829 "oxum_file_count": oxum_file_count, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
830 "oxum_byte_count": oxum_byte_count, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
831 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
832 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
833 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
834 def _validate_completeness(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
835 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
836 Verify that the actual file manifests match the files in the data directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
837 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
838 errors = list() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
839 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
840 # First we'll make sure there's no mismatch between the filesystem |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
841 # and the list of files in the manifest(s) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
842 only_in_manifests, only_on_fs = self.compare_manifests_with_fs() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
843 for path in only_in_manifests: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
844 e = FileMissing(path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
845 LOGGER.warning(force_unicode(e)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
846 errors.append(e) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
847 for path in only_on_fs: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
848 e = UnexpectedFile(path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
849 LOGGER.warning(force_unicode(e)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
850 errors.append(e) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
851 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
852 if errors: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
853 raise BagValidationError(_("Bag validation failed"), errors) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
854 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
855 def _validate_entries(self, processes): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
856 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
857 Verify that the actual file contents match the recorded hashes stored in the manifest files |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
858 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
859 errors = list() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
860 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
861 if os.name == "posix": |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
862 worker_init = posix_multiprocessing_worker_initializer |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
863 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
864 worker_init = None |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
865 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
866 args = ( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
867 ( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
868 self.path, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
869 self.normalized_filesystem_names.get(rel_path, rel_path), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
870 hashes, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
871 self.algorithms, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
872 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
873 for rel_path, hashes in self.entries.items() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
874 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
875 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
876 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
877 if processes == 1: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
878 hash_results = [_calc_hashes(i) for i in args] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
879 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
880 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
881 pool = multiprocessing.Pool( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
882 processes if processes else None, initializer=worker_init |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
883 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
884 hash_results = pool.map(_calc_hashes, args) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
885 finally: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
886 pool.terminate() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
887 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
888 # Any unhandled exceptions are probably fatal |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
889 except: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
890 LOGGER.exception(_("Unable to calculate file hashes for %s"), self) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
891 raise |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
892 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
893 for rel_path, f_hashes, hashes in hash_results: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
894 for alg, computed_hash in f_hashes.items(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
895 stored_hash = hashes[alg] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
896 if stored_hash.lower() != computed_hash: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
897 e = ChecksumMismatch( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
898 rel_path, alg, stored_hash.lower(), computed_hash |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
899 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
900 LOGGER.warning(force_unicode(e)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
901 errors.append(e) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
902 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
903 if errors: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
904 raise BagValidationError(_("Bag validation failed"), errors) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
905 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
906 def _validate_bagittxt(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
907 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
908 Verify that bagit.txt conforms to specification |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
909 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
910 bagit_file_path = os.path.join(self.path, "bagit.txt") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
911 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
912 # Note that we are intentionally opening this file in binary mode so we can confirm |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
913 # that it does not start with the UTF-8 byte-order-mark |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
914 with open(bagit_file_path, "rb") as bagit_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
915 first_line = bagit_file.read(4) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
916 if first_line.startswith(codecs.BOM_UTF8): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
917 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
918 _("bagit.txt must not contain a byte-order mark") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
919 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
920 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
921 def _path_is_dangerous(self, path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
922 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
923 Return true if path looks dangerous, i.e. potentially operates |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
924 outside the bagging directory structure, e.g. ~/.bashrc, ../../../secrets.json, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
925 \\?\c:\, D:\sys32\cmd.exe |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
926 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
927 if os.path.isabs(path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
928 return True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
929 if os.path.expanduser(path) != path: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
930 return True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
931 if os.path.expandvars(path) != path: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
932 return True |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
933 real_path = os.path.realpath(os.path.join(self.path, path)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
934 real_path = os.path.normpath(real_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
935 bag_path = os.path.realpath(self.path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
936 bag_path = os.path.normpath(bag_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
937 common = os.path.commonprefix((bag_path, real_path)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
938 return not (common == bag_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
939 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
940 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
941 class BagError(Exception): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
942 pass |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
943 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
944 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
945 class BagValidationError(BagError): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
946 def __init__(self, message, details=None): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
947 super(BagValidationError, self).__init__() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
948 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
949 if details is None: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
950 details = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
951 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
952 self.message = message |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
953 self.details = details |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
954 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
955 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
956 if len(self.details) > 0: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
957 details = "; ".join([force_unicode(e) for e in self.details]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
958 return "%s: %s" % (self.message, details) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
959 return self.message |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
960 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
961 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
962 class ManifestErrorDetail(BagError): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
963 def __init__(self, path): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
964 super(ManifestErrorDetail, self).__init__() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
965 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
966 self.path = path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
967 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
968 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
969 class ChecksumMismatch(ManifestErrorDetail): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
970 def __init__(self, path, algorithm=None, expected=None, found=None): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
971 super(ChecksumMismatch, self).__init__(path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
972 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
973 self.path = path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
974 self.algorithm = algorithm |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
975 self.expected = expected |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
976 self.found = found |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
977 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
978 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
979 return _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
980 '%(path)s %(algorithm)s validation failed: expected="%(expected)s" found="%(found)s"' |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
981 ) % { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
982 "path": force_unicode(self.path), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
983 "algorithm": self.algorithm, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
984 "expected": self.expected, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
985 "found": self.found, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
986 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
987 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
988 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
989 class FileMissing(ManifestErrorDetail): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
990 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
991 return _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
992 "%s exists in manifest but was not found on filesystem" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
993 ) % force_unicode(self.path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
994 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
995 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
996 class UnexpectedFile(ManifestErrorDetail): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
997 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
998 return _("%s exists on filesystem but is not in the manifest") % self.path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
999 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1000 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1001 class FileNormalizationConflict(BagError): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1002 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1003 Exception raised when two files differ only in normalization and thus |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1004 are not safely portable |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1005 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1006 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1007 def __init__(self, file_a, file_b): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1008 super(FileNormalizationConflict, self).__init__() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1009 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1010 self.file_a = file_a |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1011 self.file_b = file_b |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1012 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1013 def __str__(self): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1014 return _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1015 'Unicode normalization conflict for file "%(file_a)s" and "%(file_b)s"' |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1016 ) % {"file_a": self.file_a, "file_b": self.file_b} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1017 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1018 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1019 def posix_multiprocessing_worker_initializer(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1020 """Ignore SIGINT in multiprocessing workers on POSIX systems""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1021 signal.signal(signal.SIGINT, signal.SIG_IGN) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1022 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1023 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1024 # The Unicode normalization form used here doesn't matter – all we care about |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1025 # is consistency since the input value will be preserved: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1026 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1027 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1028 def normalize_unicode_py3(s): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1029 return unicodedata.normalize("NFC", s) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1030 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1031 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1032 def normalize_unicode_py2(s): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1033 if isinstance(s, str): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1034 s = s.decode("utf-8") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1035 return unicodedata.normalize("NFC", s) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1036 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1037 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1038 if sys.version_info > (3, 0): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1039 normalize_unicode = normalize_unicode_py3 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1040 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1041 normalize_unicode = normalize_unicode_py2 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1042 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1043 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1044 def build_unicode_normalized_lookup_dict(filenames): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1045 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1046 Return a dictionary mapping unicode-normalized filenames to as-encoded |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1047 values to efficiently detect conflicts between the filesystem and manifests. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1048 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1049 This is necessary because some filesystems and utilities may automatically |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1050 apply a different Unicode normalization form to filenames than was applied |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1051 when the bag was originally created. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1052 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1053 The best known example of this is when a bag is created using a |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1054 normalization form other than NFD and then transferred to a Mac where the |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1055 HFS+ filesystem will transparently normalize filenames to a variant of NFD |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1056 for every call: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1057 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1058 https://developer.apple.com/legacy/library/technotes/tn/tn1150.html#UnicodeSubtleties |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1059 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1060 Windows is documented as storing filenames exactly as provided: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1061 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1062 https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1063 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1064 Linux performs no normalization in the kernel but it is technically |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1065 valid for a filesystem to perform normalization, such as when an HFS+ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1066 volume is mounted. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1067 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1068 See http://www.unicode.org/reports/tr15/ for a full discussion of |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1069 equivalence and normalization in Unicode. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1070 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1071 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1072 output = dict() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1073 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1074 for filename in filenames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1075 normalized_filename = normalize_unicode(filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1076 if normalized_filename in output: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1077 raise FileNormalizationConflict(filename, output[normalized_filename]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1078 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1079 output[normalized_filename] = filename |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1080 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1081 return output |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1082 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1083 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1084 def get_hashers(algorithms): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1085 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1086 Given a list of algorithm names, return a dictionary of hasher instances |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1087 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1088 This avoids redundant code between the creation and validation code where in |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1089 both cases we want to avoid reading the same file more than once. The |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1090 intended use is a simple for loop: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1091 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1092 for block in file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1093 for hasher in hashers.values(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1094 hasher.update(block) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1095 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1096 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1097 hashers = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1098 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1099 for alg in algorithms: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1100 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1101 hasher = hashlib.new(alg) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1102 except ValueError: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1103 LOGGER.warning( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1104 _("Disabling requested hash algorithm %s: hashlib does not support it"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1105 alg, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1106 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1107 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1108 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1109 hashers[alg] = hasher |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1110 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1111 if not hashers: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1112 raise ValueError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1113 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1114 "Unable to continue: hashlib does not support any of the requested algorithms!" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1115 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1116 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1117 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1118 return hashers |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1119 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1120 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1121 def _calc_hashes(args): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1122 # auto unpacking of sequences illegal in Python3 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1123 (base_path, rel_path, hashes, algorithms) = args |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1124 full_path = os.path.join(base_path, rel_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1125 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1126 # Create a clone of the default empty hash objects: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1127 f_hashers = dict((alg, hashlib.new(alg)) for alg in hashes if alg in algorithms) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1128 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1129 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1130 f_hashes = _calculate_file_hashes(full_path, f_hashers) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1131 except BagValidationError as e: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1132 f_hashes = dict((alg, force_unicode(e)) for alg in f_hashers.keys()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1133 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1134 return rel_path, f_hashes, hashes |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1135 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1136 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1137 def _calculate_file_hashes(full_path, f_hashers): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1138 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1139 Returns a dictionary of (algorithm, hexdigest) values for the provided |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1140 filename |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1141 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1142 LOGGER.info(_("Verifying checksum for file %s"), full_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1143 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1144 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1145 with open(full_path, "rb") as f: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1146 while True: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1147 block = f.read(HASH_BLOCK_SIZE) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1148 if not block: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1149 break |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1150 for i in f_hashers.values(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1151 i.update(block) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1152 except (OSError, IOError) as e: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1153 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1154 _("Could not read %(filename)s: %(error)s") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1155 % {"filename": full_path, "error": force_unicode(e)} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1156 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1157 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1158 return dict((alg, h.hexdigest()) for alg, h in f_hashers.items()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1159 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1160 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1161 def _load_tag_file(tag_file_name, encoding="utf-8-sig"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1162 with open_text_file(tag_file_name, "r", encoding=encoding) as tag_file: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1163 # Store duplicate tags as list of vals |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1164 # in order of parsing under the same key. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1165 tags = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1166 for name, value in _parse_tags(tag_file): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1167 if name not in tags: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1168 tags[name] = value |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1169 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1170 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1171 if not isinstance(tags[name], list): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1172 tags[name] = [tags[name], value] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1173 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1174 tags[name].append(value) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1175 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1176 return tags |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1177 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1178 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1179 def _parse_tags(tag_file): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1180 """Parses a tag file, according to RFC 2822. This |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1181 includes line folding, permitting extra-long |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1182 field values. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1183 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1184 See http://www.faqs.org/rfcs/rfc2822.html for |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1185 more information. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1186 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1187 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1188 tag_name = None |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1189 tag_value = None |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1190 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1191 # Line folding is handled by yielding values only after we encounter |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1192 # the start of a new tag, or if we pass the EOF. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1193 for num, line in enumerate(tag_file): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1194 # Skip over any empty or blank lines. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1195 if len(line) == 0 or line.isspace(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1196 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1197 elif line[0].isspace() and tag_value is not None: # folded line |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1198 tag_value += line |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1199 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1200 # Starting a new tag; yield the last one. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1201 if tag_name: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1202 yield (tag_name, tag_value.strip()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1203 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1204 if ":" not in line: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1205 raise BagValidationError( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1206 _("%(filename)s contains invalid tag: %(line)s") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1207 % { |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1208 "line": line.strip(), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1209 "filename": os.path.basename(tag_file.name), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1210 } |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1211 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1212 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1213 parts = line.strip().split(":", 1) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1214 tag_name = parts[0].strip() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1215 tag_value = parts[1] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1216 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1217 # Passed the EOF. All done after this. |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1218 if tag_name: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1219 yield (tag_name, tag_value.strip()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1220 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1221 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1222 def _make_tag_file(bag_info_path, bag_info): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1223 headers = sorted(bag_info.keys()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1224 with open_text_file(bag_info_path, "w") as f: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1225 for h in headers: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1226 values = bag_info[h] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1227 if not isinstance(values, list): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1228 values = [values] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1229 for txt in values: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1230 # strip CR, LF and CRLF so they don't mess up the tag file |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1231 txt = re.sub(r"\n|\r|(\r\n)", "", force_unicode(txt)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1232 f.write("%s: %s\n" % (h, txt)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1233 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1234 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1235 def make_manifests(data_dir, processes, algorithms=DEFAULT_CHECKSUMS, encoding="utf-8"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1236 LOGGER.info( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1237 _("Using %(process_count)d processes to generate manifests: %(algorithms)s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1238 {"process_count": processes, "algorithms": ", ".join(algorithms)}, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1239 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1240 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1241 manifest_line_generator = partial(generate_manifest_lines, algorithms=algorithms) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1242 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1243 if processes > 1: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1244 pool = multiprocessing.Pool(processes=processes) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1245 checksums = pool.map(manifest_line_generator, _walk(data_dir)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1246 pool.close() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1247 pool.join() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1248 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1249 checksums = [manifest_line_generator(i) for i in _walk(data_dir)] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1250 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1251 # At this point we have a list of tuples which start with the algorithm name: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1252 manifest_data = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1253 for batch in checksums: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1254 for entry in batch: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1255 manifest_data.setdefault(entry[0], []).append(entry[1:]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1256 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1257 # These will be keyed on the algorithm name so we can perform sanity checks |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1258 # below to catch failures in the hashing process: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1259 num_files = defaultdict(lambda: 0) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1260 total_bytes = defaultdict(lambda: 0) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1261 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1262 for algorithm, values in manifest_data.items(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1263 manifest_filename = "manifest-%s.txt" % algorithm |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1264 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1265 with open_text_file(manifest_filename, "w", encoding=encoding) as manifest: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1266 for digest, filename, byte_count in values: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1267 manifest.write("%s %s\n" % (digest, _encode_filename(filename))) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1268 num_files[algorithm] += 1 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1269 total_bytes[algorithm] += byte_count |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1270 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1271 # We'll use sets of the values for the error checks and eventually return the payload oxum values: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1272 byte_value_set = set(total_bytes.values()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1273 file_count_set = set(num_files.values()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1274 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1275 # allow a bag with an empty payload |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1276 if not byte_value_set and not file_count_set: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1277 return 0, 0 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1278 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1279 if len(file_count_set) != 1: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1280 raise RuntimeError(_("Expected the same number of files for each checksum")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1281 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1282 if len(byte_value_set) != 1: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1283 raise RuntimeError(_("Expected the same number of bytes for each checksums")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1284 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1285 return byte_value_set.pop(), file_count_set.pop() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1286 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1287 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1288 def _make_tagmanifest_file(alg, bag_dir, encoding="utf-8"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1289 tagmanifest_file = join(bag_dir, "tagmanifest-%s.txt" % alg) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1290 LOGGER.info(_("Creating %s"), tagmanifest_file) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1291 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1292 checksums = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1293 for f in _find_tag_files(bag_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1294 if re.match(r"^tagmanifest-.+\.txt$", f): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1295 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1296 with open(join(bag_dir, f), "rb") as fh: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1297 m = hashlib.new(alg) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1298 while True: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1299 block = fh.read(HASH_BLOCK_SIZE) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1300 if not block: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1301 break |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1302 m.update(block) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1303 checksums.append((m.hexdigest(), f)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1304 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1305 with open_text_file( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1306 join(bag_dir, tagmanifest_file), mode="w", encoding=encoding |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1307 ) as tagmanifest: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1308 for digest, filename in checksums: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1309 tagmanifest.write("%s %s\n" % (digest, filename)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1310 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1311 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1312 def _find_tag_files(bag_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1313 for dir in os.listdir(bag_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1314 if dir != "data": |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1315 if os.path.isfile(dir) and not dir.startswith("tagmanifest-"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1316 yield dir |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1317 for dir_name, _, filenames in os.walk(dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1318 for filename in filenames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1319 if filename.startswith("tagmanifest-"): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1320 continue |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1321 # remove everything up to the bag_dir directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1322 p = join(dir_name, filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1323 yield os.path.relpath(p, bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1324 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1325 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1326 def _walk(data_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1327 for dirpath, dirnames, filenames in os.walk(data_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1328 # if we don't sort here the order of entries is non-deterministic |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1329 # which makes it hard to test the fixity of tagmanifest-md5.txt |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1330 filenames.sort() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1331 dirnames.sort() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1332 for fn in filenames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1333 path = os.path.join(dirpath, fn) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1334 # BagIt spec requires manifest to always use '/' as path separator |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1335 if os.path.sep != "/": |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1336 parts = path.split(os.path.sep) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1337 path = "/".join(parts) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1338 yield path |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1339 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1340 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1341 def _can_bag(test_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1342 """Scan the provided directory for files which cannot be bagged due to insufficient permissions""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1343 unbaggable = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1344 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1345 if not os.access(test_dir, os.R_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1346 # We cannot continue without permission to read the source directory |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1347 unbaggable.append(test_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1348 return unbaggable |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1349 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1350 if not os.access(test_dir, os.W_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1351 unbaggable.append(test_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1352 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1353 for dirpath, dirnames, filenames in os.walk(test_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1354 for directory in dirnames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1355 full_path = os.path.join(dirpath, directory) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1356 if not os.access(full_path, os.W_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1357 unbaggable.append(full_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1358 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1359 return unbaggable |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1360 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1361 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1362 def _can_read(test_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1363 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1364 returns ((unreadable_dirs), (unreadable_files)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1365 """ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1366 unreadable_dirs = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1367 unreadable_files = [] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1368 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1369 if not os.access(test_dir, os.R_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1370 unreadable_dirs.append(test_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1371 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1372 for dirpath, dirnames, filenames in os.walk(test_dir): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1373 for dn in dirnames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1374 full_path = os.path.join(dirpath, dn) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1375 if not os.access(full_path, os.R_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1376 unreadable_dirs.append(full_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1377 for fn in filenames: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1378 full_path = os.path.join(dirpath, fn) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1379 if not os.access(full_path, os.R_OK): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1380 unreadable_files.append(full_path) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1381 return (tuple(unreadable_dirs), tuple(unreadable_files)) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1382 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1383 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1384 def generate_manifest_lines(filename, algorithms=DEFAULT_CHECKSUMS): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1385 LOGGER.info(_("Generating manifest lines for file %s"), filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1386 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1387 # For performance we'll read the file only once and pass it block |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1388 # by block to every requested hash algorithm: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1389 hashers = get_hashers(algorithms) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1390 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1391 total_bytes = 0 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1392 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1393 with open(filename, "rb") as f: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1394 while True: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1395 block = f.read(HASH_BLOCK_SIZE) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1396 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1397 if not block: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1398 break |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1399 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1400 total_bytes += len(block) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1401 for hasher in hashers.values(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1402 hasher.update(block) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1403 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1404 decoded_filename = _decode_filename(filename) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1405 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1406 # We'll generate a list of results in roughly manifest format but prefixed with the algorithm: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1407 results = [ |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1408 (alg, hasher.hexdigest(), decoded_filename, total_bytes) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1409 for alg, hasher in hashers.items() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1410 ] |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1411 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1412 return results |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1413 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1414 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1415 def _encode_filename(s): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1416 s = s.replace("\r", "%0D") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1417 s = s.replace("\n", "%0A") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1418 return s |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1419 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1420 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1421 def _decode_filename(s): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1422 s = re.sub(r"%0D", "\r", s, re.IGNORECASE) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1423 s = re.sub(r"%0A", "\n", s, re.IGNORECASE) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1424 return s |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1425 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1426 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1427 def force_unicode_py2(s): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1428 """Reliably return a Unicode string given a possible unicode or byte string""" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1429 if isinstance(s, str): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1430 return s.decode("utf-8") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1431 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1432 return unicode(s) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1433 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1434 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1435 if sys.version_info > (3, 0): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1436 force_unicode = str |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1437 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1438 force_unicode = force_unicode_py2 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1439 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1440 # following code is used for command line program |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1441 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1442 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1443 class BagArgumentParser(argparse.ArgumentParser): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1444 def __init__(self, *args, **kwargs): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1445 self.bag_info = {} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1446 argparse.ArgumentParser.__init__(self, *args, **kwargs) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1447 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1448 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1449 class BagHeaderAction(argparse.Action): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1450 def __call__(self, parser, _, values, option_string=None): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1451 opt = option_string.lstrip("--") |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1452 opt_caps = "-".join([o.capitalize() for o in opt.split("-")]) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1453 parser.bag_info[opt_caps] = values |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1454 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1455 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1456 def _make_parser(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1457 parser = BagArgumentParser( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1458 formatter_class=argparse.RawDescriptionHelpFormatter, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1459 description="bagit-python version %s\n\n%s\n" % (VERSION, __doc__.strip()), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1460 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1461 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1462 "--processes", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1463 type=int, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1464 dest="processes", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1465 default=1, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1466 help=_( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1467 "Use multiple processes to calculate checksums faster (default: %(default)s)" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1468 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1469 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1470 parser.add_argument("--log", help=_("The name of the log file (default: stdout)")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1471 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1472 "--quiet", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1473 action="store_true", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1474 help=_("Suppress all progress information other than errors"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1475 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1476 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1477 "--validate", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1478 action="store_true", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1479 help=_( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1480 "Validate existing bags in the provided directories instead of" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1481 " creating new ones" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1482 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1483 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1484 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1485 "--fast", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1486 action="store_true", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1487 help=_( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1488 "Modify --validate behaviour to only test whether the bag directory" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1489 " has the number of files and total size specified in Payload-Oxum" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1490 " without performing checksum validation to detect corruption." |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1491 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1492 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1493 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1494 "--completeness-only", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1495 action="store_true", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1496 help=_( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1497 "Modify --validate behaviour to test whether the bag directory" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1498 " has the expected payload specified in the checksum manifests" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1499 " without performing checksum validation to detect corruption." |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1500 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1501 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1502 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1503 checksum_args = parser.add_argument_group( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1504 _("Checksum Algorithms"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1505 _( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1506 "Select the manifest algorithms to be used when creating bags" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1507 " (default=%s)" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1508 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1509 % ", ".join(DEFAULT_CHECKSUMS), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1510 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1511 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1512 for i in CHECKSUM_ALGOS: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1513 alg_name = re.sub(r"^([A-Z]+)(\d+)$", r"\1-\2", i.upper()) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1514 checksum_args.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1515 "--%s" % i, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1516 action="append_const", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1517 dest="checksums", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1518 const=i, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1519 help=_("Generate %s manifest when creating a bag") % alg_name, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1520 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1521 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1522 metadata_args = parser.add_argument_group(_("Optional Bag Metadata")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1523 for header in STANDARD_BAG_INFO_HEADERS: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1524 metadata_args.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1525 "--%s" % header.lower(), type=str, action=BagHeaderAction |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1526 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1527 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1528 parser.add_argument( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1529 "directory", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1530 nargs="+", |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1531 help=_( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1532 "Directory which will be converted into a bag in place" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1533 " by moving any existing files into the BagIt structure" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1534 " and creating the manifests and other metadata." |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1535 ), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1536 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1537 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1538 return parser |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1539 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1540 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1541 def _configure_logging(opts): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1542 log_format = "%(asctime)s - %(levelname)s - %(message)s" |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1543 if opts.quiet: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1544 level = logging.ERROR |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1545 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1546 level = logging.INFO |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1547 if opts.log: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1548 logging.basicConfig(filename=opts.log, level=level, format=log_format) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1549 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1550 logging.basicConfig(level=level, format=log_format) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1551 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1552 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1553 def main(): |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1554 if "--version" in sys.argv: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1555 print(_("bagit-python version %s") % VERSION) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1556 sys.exit(0) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1557 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1558 parser = _make_parser() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1559 args = parser.parse_args() |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1560 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1561 if args.processes < 0: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1562 parser.error(_("The number of processes must be 0 or greater")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1563 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1564 if args.fast and not args.validate: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1565 parser.error(_("--fast is only allowed as an option for --validate!")) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1566 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1567 _configure_logging(args) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1568 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1569 rc = 0 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1570 for bag_dir in args.directory: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1571 # validate the bag |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1572 if args.validate: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1573 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1574 bag = Bag(bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1575 # validate throws a BagError or BagValidationError |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1576 bag.validate( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1577 processes=args.processes, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1578 fast=args.fast, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1579 completeness_only=args.completeness_only, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1580 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1581 if args.fast: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1582 LOGGER.info(_("%s valid according to Payload-Oxum"), bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1583 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1584 LOGGER.info(_("%s is valid"), bag_dir) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1585 except BagError as e: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1586 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1587 _("%(bag)s is invalid: %(error)s"), {"bag": bag_dir, "error": e} |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1588 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1589 rc = 1 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1590 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1591 # make the bag |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1592 else: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1593 try: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1594 make_bag( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1595 bag_dir, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1596 bag_info=parser.bag_info, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1597 processes=args.processes, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1598 checksums=args.checksums, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1599 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1600 except Exception as exc: |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1601 LOGGER.error( |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1602 _("Failed to create bag in %(bag_directory)s: %(error)s"), |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1603 {"bag_directory": bag_dir, "error": exc}, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1604 exc_info=True, |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1605 ) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1606 rc = 1 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1607 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1608 sys.exit(rc) |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1609 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1610 |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1611 if __name__ == "__main__": |
26e78fe6e8c4
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
shellac
parents:
diff
changeset
|
1612 main() |