From 6c4a69c8fc0635140ed2cf589cb87397cd9e8e87 Mon Sep 17 00:00:00 2001 From: Kyle K Date: Mon, 27 Mar 2017 03:14:49 -0500 Subject: use templating to render HTML page on /app --- README.md | 27 +++++++++++++++++++++++++++ cl.py | 8 ++++---- django_clscrap/settings.py | 1 - myapp/templates/index.html | 37 +++++++++++++++++++++++++++++++++++++ myapp/views.py | 4 +++- requirements.txt | 1 - screenshot.jpg | Bin 0 -> 110832 bytes 7 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 README.md create mode 100644 myapp/templates/index.html create mode 100644 screenshot.jpg diff --git a/README.md b/README.md new file mode 100644 index 0000000..1eb0fff --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# CLscrap + +CLscrap is a craigslist scraper that provides a RESTful API at /api/cl that returns JSON data + + - uses Django and templating + - renders results at /app + +### Installation + +CLscrap depends on following modules + - beautifulsoup4==4.5.3 + - bs4==0.0.1 + - Django==1.10.6 + - requests==2.13.0 + +Install +```sh +$ pip install -r requirements.txt +$ python manage.py migrate +$ python manage.py createsuperuser +$ python manage.py runserver +$ curl http://localhost:8000/api/cl +``` + +### Screenshot + +![clscrap](https://raw.githubusercontent.com/fatalhalt/CLscrap/master/screenshot.jpg?raw=true) diff --git a/cl.py b/cl.py index fafd65b..a908a99 100644 --- a/cl.py +++ b/cl.py @@ -6,8 +6,8 @@ from bs4 import BeautifulSoup def query_craigslist(baseurl=None, keyword='wrx|sti'): if baseurl is None: - baseurl = 'https://chicago.craigslist.org/' - response = requests.get(baseurl + 'search/pta', params={'query': keyword, 'srchType': 'T'}) + baseurl = 'https://chicago.craigslist.org' + response = requests.get(baseurl + '/search/pta', params={'query': keyword, 'srchType': 'T'}) soup = BeautifulSoup(response.content, "html.parser") results = soup.find_all('li', {'class': 'result-row'}) # at max 120 results per 1 page @@ -27,12 +27,12 @@ def query_craigslist(baseurl=None, keyword='wrx|sti'): except AttributeError: pass # ignore empty fields - return items + return items, baseurl, keyword def main(): parser = argparse.ArgumentParser(description="craigslist WRX and STi parts finder", parents=()) - parser.add_argument("-b", "--baseurl", help='baseurl, e.g. https://chicago.craigslist.org/') + parser.add_argument("-b", "--baseurl", help='baseurl, e.g. https://chicago.craigslist.org') parser.add_argument("-k", "--keyword", default='wrx|sti', help='keyword to search') args, extra_args = parser.parse_known_args() diff --git a/django_clscrap/settings.py b/django_clscrap/settings.py index 39a1021..5c8092f 100644 --- a/django_clscrap/settings.py +++ b/django_clscrap/settings.py @@ -37,7 +37,6 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'rest_framework', 'myapp.apps.MyappConfig', ] diff --git a/myapp/templates/index.html b/myapp/templates/index.html new file mode 100644 index 0000000..1a1f618 --- /dev/null +++ b/myapp/templates/index.html @@ -0,0 +1,37 @@ + + + + + Craigslist Scraper + + + +

Craigslist Scrapper

+get list of current WRX STi parts for sale in Chicago area +
+    url: {{ baseurl }}
+    query: {{ keyword }}
+
+    author: fatalhalt
+    https://github.com/fatalhalt/CLscrap
+
+
+
+{% for i in data %} +
+ {{ i.title }} + {{ i.date }} + {{ i.price }} + {{ i.hood }} + img +
+{% endfor %} +
+ + \ No newline at end of file diff --git a/myapp/views.py b/myapp/views.py index 6b014ac..600a061 100644 --- a/myapp/views.py +++ b/myapp/views.py @@ -2,11 +2,13 @@ from django.shortcuts import render # Create your views here. from django.http import HttpResponse, JsonResponse +from django.shortcuts import render import cl def app(request): - return HttpResponse("hello, app!") + data, baseurl, keyword = cl.query_craigslist() + return render(request, 'index.html', {'data': data, 'baseurl': baseurl, 'keyword': keyword}) def page(request): diff --git a/requirements.txt b/requirements.txt index d6448fc..df760c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ beautifulsoup4==4.5.3 bs4==0.0.1 Django==1.10.6 -djangorestframework==3.6.2 requests==2.13.0 diff --git a/screenshot.jpg b/screenshot.jpg new file mode 100644 index 0000000..9f36f13 Binary files /dev/null and b/screenshot.jpg differ -- cgit v1.2.3