From eb98f903635b2b10cc40271e58591e522226784d Mon Sep 17 00:00:00 2001 From: Kyle K Date: Sun, 26 Mar 2017 22:03:45 -0500 Subject: initial commit --- .gitignore | 26 +++++++++ cl.py | 47 +++++++++++++++++ django_clscrap/__init__.py | 0 django_clscrap/settings.py | 122 +++++++++++++++++++++++++++++++++++++++++++ django_clscrap/urls.py | 24 +++++++++ django_clscrap/wsgi.py | 16 ++++++ manage.py | 22 ++++++++ myapp/__init__.py | 0 myapp/admin.py | 3 ++ myapp/apps.py | 5 ++ myapp/migrations/__init__.py | 0 myapp/models.py | 3 ++ myapp/tests.py | 3 ++ myapp/urls.py | 8 +++ myapp/views.py | 12 +++++ pyvenv.cfg | 3 ++ requirements.txt | 5 ++ 17 files changed, 299 insertions(+) create mode 100644 .gitignore create mode 100644 cl.py create mode 100644 django_clscrap/__init__.py create mode 100644 django_clscrap/settings.py create mode 100644 django_clscrap/urls.py create mode 100644 django_clscrap/wsgi.py create mode 100644 manage.py create mode 100644 myapp/__init__.py create mode 100644 myapp/admin.py create mode 100644 myapp/apps.py create mode 100644 myapp/migrations/__init__.py create mode 100644 myapp/models.py create mode 100644 myapp/tests.py create mode 100644 myapp/urls.py create mode 100644 myapp/views.py create mode 100644 pyvenv.cfg create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..caa80f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# If you need to exclude files such as those generated by an IDE, use +# $GIT_DIR/info/exclude or the core.excludesFile configuration variable as +# described in https://git-scm.com/docs/gitignore + +*.egg-info +*.pot +*.py[co] +.tox/ +__pycache__ +MANIFEST +dist/ +docs/_build/ +docs/locale/ +node_modules/ +tests/coverage_html/ +tests/.coverage +build/ +tests/report/ + +Include/ +Lib/ +Scripts/ + +.idea/ + +db.sqlite3 diff --git a/cl.py b/cl.py new file mode 100644 index 0000000..fafd65b --- /dev/null +++ b/cl.py @@ -0,0 +1,47 @@ +import argparse +import sys +import requests +from bs4 import BeautifulSoup + + +def query_craigslist(baseurl=None, keyword='wrx|sti'): + if baseurl is None: + baseurl = 'https://chicago.craigslist.org/' + response = requests.get(baseurl + 'search/pta', params={'query': keyword, 'srchType': 'T'}) + soup = BeautifulSoup(response.content, "html.parser") + + results = soup.find_all('li', {'class': 'result-row'}) # at max 120 results per 1 page + items = [] + + for i in results: + try: + title = i.find('a', {'class': 'result-title hdrlnk'}).get_text() + link = i.find('a', {'class': 'result-title hdrlnk'}).get('href') + date = i.find('time', {'class': 'result-date'}).get('datetime') + price = i.find('span', {'class': 'result-price'}).get_text() + hood = i.find('span', {'class': 'result-hood'}).get_text() + imgs = i.find('a', {'class': 'result-image gallery'}).get('data-ids').split(',') + img_1 = 'https://images.craigslist.org/' + imgs[0][2:] + '_300x300.jpg' + d = {'title': title, 'link': link, 'date': date, 'price': price, 'hood': hood, 'img': img_1} + items.append(d) + except AttributeError: + pass # ignore empty fields + + return items + + +def main(): + parser = argparse.ArgumentParser(description="craigslist WRX and STi parts finder", parents=()) + parser.add_argument("-b", "--baseurl", help='baseurl, e.g. https://chicago.craigslist.org/') + parser.add_argument("-k", "--keyword", default='wrx|sti', help='keyword to search') + + args, extra_args = parser.parse_known_args() + partlist = query_craigslist(args.baseurl, args.keyword) + print(partlist) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + pass diff --git a/django_clscrap/__init__.py b/django_clscrap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_clscrap/settings.py b/django_clscrap/settings.py new file mode 100644 index 0000000..27cb351 --- /dev/null +++ b/django_clscrap/settings.py @@ -0,0 +1,122 @@ +""" +Django settings for django_clscrap project. + +Generated by 'django-admin startproject' using Django 1.10.6. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.10/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '-(n)6%3v7^c-nbqc%re3@12n@!buzm(+i!ihqs=14#udznc8f@' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + # 'myapp.apps.MyappConfig' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'django_clscrap.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(BASE_DIR, 'templates')] + , + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'django_clscrap.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.10/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.10/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.10/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/django_clscrap/urls.py b/django_clscrap/urls.py new file mode 100644 index 0000000..66dca1d --- /dev/null +++ b/django_clscrap/urls.py @@ -0,0 +1,24 @@ +"""django_clscrap URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import include, url +from django.contrib import admin +from django.http import HttpResponseRedirect + +urlpatterns = [ + url(r'^$', lambda r: HttpResponseRedirect('app/')), # 302 redirect from / to app/index + url(r'^admin/', admin.site.urls), + url(r'^app/', include('myapp.urls')), +] diff --git a/django_clscrap/wsgi.py b/django_clscrap/wsgi.py new file mode 100644 index 0000000..16e14e0 --- /dev/null +++ b/django_clscrap/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for django_clscrap project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_clscrap.settings") + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..b8d661b --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_clscrap.settings") + try: + from django.core.management import execute_from_command_line + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise + execute_from_command_line(sys.argv) diff --git a/myapp/__init__.py b/myapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myapp/admin.py b/myapp/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/myapp/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/myapp/apps.py b/myapp/apps.py new file mode 100644 index 0000000..74d6d13 --- /dev/null +++ b/myapp/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MyappConfig(AppConfig): + name = 'myapp' diff --git a/myapp/migrations/__init__.py b/myapp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myapp/models.py b/myapp/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/myapp/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/myapp/tests.py b/myapp/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/myapp/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/myapp/urls.py b/myapp/urls.py new file mode 100644 index 0000000..0ef0e5a --- /dev/null +++ b/myapp/urls.py @@ -0,0 +1,8 @@ +from django.conf.urls import url + +from . import views + +urlpatterns = [ + url(r'^$', views.app, name='app'), # at this point /app string is consumed + url(r'^page/', views.page, name='page'), # at this point /app string is consumed as well, therefore /app/page hits this +] \ No newline at end of file diff --git a/myapp/views.py b/myapp/views.py new file mode 100644 index 0000000..10b5c99 --- /dev/null +++ b/myapp/views.py @@ -0,0 +1,12 @@ +from django.shortcuts import render + +# Create your views here. +from django.http import HttpResponse + + +def app(request): + return HttpResponse("hello, app!") + + +def page(request): + return HttpResponse("hello, page!") \ No newline at end of file diff --git a/pyvenv.cfg b/pyvenv.cfg new file mode 100644 index 0000000..43a8395 --- /dev/null +++ b/pyvenv.cfg @@ -0,0 +1,3 @@ +home = C:\Program Files\Python36 +include-system-site-packages = false +version = 3.6.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d6448fc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +beautifulsoup4==4.5.3 +bs4==0.0.1 +Django==1.10.6 +djangorestframework==3.6.2 +requests==2.13.0 -- cgit v1.2.3