Skip to content

Commit aac0433

Browse files
authored
Merge pull request #34 from ocefpaf/fix_py2k_syntax
Fix python 2 syntax in the README
2 parents 60cab27 + 6ded170 commit aac0433

File tree

1 file changed

+27
-13
lines changed

1 file changed

+27
-13
lines changed

README.md

+27-13
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ conda install thredds_crawler --channel conda-forge
2121

2222
### Select
2323

24-
You can select datasets based on their THREDDS ID using the "select" parameter. Python regex is supported.
24+
You can select datasets based on their THREDDS ID using the "select" parameter.
25+
Python regex is supported.
2526

2627
```python
2728
from thredds_crawler.crawl import Crawl
29+
2830
c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"])
29-
print c.datasets
31+
print(c.datasets)
3032
[
3133
<LeafDataset id: MODIS-Agg, name: MODIS-Complete Aggregation, services: ["OPENDAP", "ISO"]>,
3234
<LeafDataset id: MODIS-2009-Agg, name: MODIS-2009 Aggregation, services: ["OPENDAP", "ISO"]>,
@@ -42,7 +44,8 @@ print c.datasets
4244

4345
### Skip
4446

45-
You can skip datasets based on their `name` and catalogRefs based on their `xlink:title`. By default, the crawler uses some common regular expressions to skip lists of thousands upon thousands of individual files that are part of aggregations or FMRCs:
47+
You can skip datasets based on their `name` and catalogRefs based on their `xlink:title`.
48+
By default, the crawler uses some common regular expressions to skip lists of thousands upon thousands of individual files that are part of aggregations or FMRCs:
4649

4750
* `.*files.*`
4851
* `.*Individual Files.*`
@@ -57,7 +60,8 @@ You can access the default `skip` list through the Crawl.SKIPS class variable
5760

5861
```python
5962
from thredds_crawler.crawl import Crawl
60-
print Crawl.SKIPS
63+
64+
print(Crawl.SKIPS)
6165
[
6266
".*files.*",
6367
".*Individual Files.*",
@@ -72,13 +76,14 @@ If you need to remove or add a new `skip`, it is **strongly** encouraged you use
7276

7377
```python
7478
from thredds_crawler.crawl import Crawl
79+
7580
skips = Crawl.SKIPS + [".*-Day-Aggregation"]
7681
c = Crawl(
7782
"http://tds.maracoos.org/thredds/MODIS.xml",
7883
select=[".*-Agg"],
7984
skip=skips
8085
)
81-
print c.datasets
86+
print(c.datasets)
8287

8388
[
8489
<LeafDataset id: MODIS-Agg, name: MODIS-Complete Aggregation, services: ["OPENDAP", "ISO"]>,
@@ -104,7 +109,7 @@ def timeit(name):
104109
startTime = time.time()
105110
yield
106111
elapsedTime = time.time() - startTime
107-
print("[{}] finished in {} ms".format(name, int(elapsedTime * 1000)))
112+
print(f"[{name}] finished in {int(elapsedTime * 1000)} ms")
108113

109114
for x in range(1, 11):
110115
with timeit("{} workers".format(x)):
@@ -204,6 +209,7 @@ logger, **do not** include `debug=True` when initializing the Crawl object.
204209

205210
```python
206211
import logging
212+
207213
crawl_log = logging.getLogger("thredds_crawler")
208214
crawl_log.setLevel(logging.WARNING)
209215
```
@@ -215,13 +221,16 @@ You can get some basic information about a LeafDataset, including the services a
215221

216222
```python
217223
from thredds_crawler.crawl import Crawl
224+
218225
c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"])
219226
dataset = c.datasets[0]
220-
print dataset.id
227+
print(dataset.id)
221228
MODIS-Agg
222-
print dataset.name
229+
230+
print(dataset.name)
223231
MODIS-Complete Aggregation
224-
print dataset.services
232+
233+
print(dataset.services)
225234
[
226235
{
227236
"url": "http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc",
@@ -240,9 +249,10 @@ If you have a list of datasets you can easily return all endpoints of a certain
240249

241250
```python
242251
from thredds_crawler.crawl import Crawl
252+
243253
c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"])
244254
urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"]
245-
print urls
255+
print(urls)
246256
[
247257
"http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc",
248258
"http://tds.maracoos.org/thredds/dodsC/MODIS-2009-Agg.nc",
@@ -262,12 +272,13 @@ This isn"t necessarialy the size on disk, because it does not account for `missi
262272

263273
```python
264274
from thredds_crawler.crawl import Crawl
275+
265276
c = Crawl(
266277
"http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html",
267278
select=["MB_.*"]
268279
)
269280
sizes = [d.size for d in c.datasets]
270-
print sizes
281+
print(sizes)
271282
[29247.410283999998, 72166.289680000002]
272283
```
273284

@@ -278,9 +289,11 @@ The entire THREDDS catalog metadata record is saved along with the dataset objec
278289

279290
```python
280291
from thredds_crawler.crawl import Crawl
292+
281293
c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"])
282294
dataset = c.datasets[0]
283-
print dataset.metadata.find("{http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0}documentation").text
295+
296+
print(dataset.metadata.find("{http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0}documentation").text)
284297
Ocean Color data are provided as a service to the broader community, and can be
285298
influenced by sensor degradation and or algorithm changes. We make efforts to keep
286299
this dataset updated and calibrated. The products in these files are experimental.
@@ -301,6 +314,7 @@ from thredds_crawler.crawl import Crawl
301314

302315
import logging
303316
import logging.handlers
317+
304318
logger = logging.getLogger("thredds_crawler")
305319
fh = logging.handlers.RotatingFileHandler("/var/log/iso_harvest/iso_harvest.log", maxBytes=1024*1024*10, backupCount=5)
306320
fh.setLevel(logging.DEBUG)
@@ -313,7 +327,7 @@ logger.addHandler(fh)
313327
logger.addHandler(ch)
314328
logger.setLevel(logging.DEBUG)
315329

316-
SAVE_DIR="/srv/http/iso"
330+
SAVE_DIR = "/srv/http/iso"
317331

318332
THREDDS_SERVERS = {
319333
"aoos": "http://thredds.axiomalaska.com/thredds/catalogs/aoos.html",

0 commit comments

Comments
 (0)