3pp-mirror/public-apis
1
0
Fork 0
mirror of https://github.com/public-apis/public-apis.git synced 2025-05-11 08:06:46 +02:00
Code Issues Projects Releases Packages Wiki Activity

Check links on PRs and Update main build script

Browse source
This commit is contained in:
Dave Machado 2017-12-28 23:44:24 -05:00
parent 2ceb3025ba
commit 21eb114d6d
No known key found for this signature in database
GPG key ID: 948D4778D01A7B3F
2 changed files with 58 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

86
build/main.sh
View file

@ -1,45 +1,55 @@
#!/bin/bash
FORMAT_FILE=../README.md
if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
echo "running on $TRAVIS_BRANCH branch"
LINK_FILE=../README.md
else
echo "running on Pull Request #$TRAVIS_PULL_REQUEST"
DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff"
curl $DIFF_URL > diff.txt
echo "------- BEGIN DIFF -------"
cat diff.txt
echo "-------- END DIFF --------"
cat diff.txt | egrep "\+" > additions.txt
echo "------ BEGIN ADDITIONS -----"
cat additions.txt
echo "------- END ADDITIONS ------"
LINK_FILE=additions.txt
echo "checking if /json was changed..."
if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then
echo "JSON files are auto-generated! Please do not update these files:"
cat json.txt
exit 1
else
echo "/json check passed!"
rm json.txt
fi
fi
echo "running format validation..."
./validate_format.py $FORMAT_FILE
if [[ $? != 0 ]]; then
echo "format validation failed!"
exit 1
else
echo "format validation passed!"
./build.sh && ./deploy.sh
if [[ $? != 0 ]]; then
echo "JSON build and deploy failed!"
else
echo "JSON build and deploy success!"
fi
echo "format validation failed!"
exit 1
fi
echo "format validation passed!"
./build.sh
if [[ $? != 0 ]]; then
echo "JSON build failed!"
else
echo "JSON build success!"
fi
if [ "$TRAVIS_BRANCH" == "master" ]
then
echo "Master build - deploying JSON"
./deploy.sh
fi
if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
echo "running on $TRAVIS_BRANCH branch - skipping Pull Request logic"
exit 0
fi
echo "running on Pull Request #$TRAVIS_PULL_REQUEST"
DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff"
curl $DIFF_URL > diff.txt
echo "------- BEGIN DIFF -------"
cat diff.txt
echo "-------- END DIFF --------"
cat diff.txt | egrep "\+" > additions.txt
echo "------ BEGIN ADDITIONS -----"
cat additions.txt
echo "------- END ADDITIONS ------"
LINK_FILE=additions.txt
echo "checking if /json was changed..."
if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then
echo "JSON files are auto-generated! Please do not update these files:"
cat json.txt
exit 1
else
echo "/json check passed!"
rm json.txt
fi
echo "running link validation..."
./validate_links.py $LINK_FILE
if [[ $? != 0 ]]; then
echo "link validation failed!"
exit 1
else
echo "link validation passed!"
fi

26
build/validate_links.py
View file

@ -1,32 +1,26 @@
#!/usr/bin/env python3
import httplib2
import json
import re
import socket
import sys
def parse_links(filename):
"""Returns a list of links from JSON object"""
data = json.load(open(filename))
links = []
for entry in data['entries']:
link = entry['Link']
https = True if link.startswith('https') else False
x = {
'link': link,
'https': https,
}
links.append(x)
"""Returns a list of URLs from text file"""
with open(filename) as fp:
data = fp.read()
raw_links = re.findall( \
'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', \
data)
links = [raw_link.replace(')', '') for raw_link in raw_links]
return links
def validate_links(links):
"""Checks each entry in JSON file for live link"""
print('Validating {} links...'.format(len(links)))
errors = []
for each in links:
link = each['link']
for link in links:
h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=5)
try:
resp = h.request(link, 'HEAD')
@ -43,7 +37,7 @@ def validate_links(links):
if __name__ == "__main__":
num_args = len(sys.argv)
if num_args < 2:
print("No .json file passed")
print("No .md file passed")
sys.exit(1)
errors = validate_links(parse_links(sys.argv[1]))
if len(errors) > 0: