diff --git a/db.sqlite3.pre_importbkup b/db.sqlite3.pre_importbkup new file mode 100644 index 0000000..4d8e72d Binary files /dev/null and b/db.sqlite3.pre_importbkup differ diff --git a/historic_data_import/__init__.py b/historic_data_import/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/historic_data_import/admin.py b/historic_data_import/admin.py new file mode 100644 index 0000000..846f6b4 --- /dev/null +++ b/historic_data_import/admin.py @@ -0,0 +1 @@ +# Register your models here. diff --git a/historic_data_import/apps.py b/historic_data_import/apps.py new file mode 100644 index 0000000..c502e41 --- /dev/null +++ b/historic_data_import/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class HistoricDataImportConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "historic_data_import" diff --git a/standalone_modules/temperature_module/data/shed-pi-2024-04-02.log b/historic_data_import/data/shed-pi-2024-04-02.log similarity index 100% rename from standalone_modules/temperature_module/data/shed-pi-2024-04-02.log rename to historic_data_import/data/shed-pi-2024-04-02.log diff --git a/historic_data_import/management/__init__.py b/historic_data_import/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/historic_data_import/management/commands/__init__.py b/historic_data_import/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/historic_data_import/management/commands/data_import.py b/historic_data_import/management/commands/data_import.py new file mode 100644 index 0000000..b25075f --- /dev/null +++ b/historic_data_import/management/commands/data_import.py @@ -0,0 +1,236 @@ +from typing import Optional + +from django.core.management.base import BaseCommand +from django.db import IntegrityError, transaction + +from shedpi_hub_dashboard.models import Device, DeviceModule, DeviceModuleReading + +# TODO: Logger + +""" +A script ot import the data that has already been gathered + +Historic data structure: + "Shed pi started: {get_time()}, using version: 0.0.1" + "Pi temp: {pi_temp}, probe_1 temp: {probe_1_temp}" + +FIXME: + Allow the data submission endpoint to take multiple device modules and readings. + Example: Pi has it's own data and so does the temp probe. More effecient to send at once + +TODO: + Module code + - Need a device module for PI temp amd Probe temp, with working api connection + - Call api endpoints to store + - probe temp + - device tenp + - device on / off (ignore datetime from the device which will be wrong on startup) + Hub code + - Need the ability to be able to create an action for device on off + + django admin device_module_readings: filter by device_module, + could be seriously heavy query, separate ticket for perf + + Allow multiple imports of different modules from the same api endpoint (low priority, greener, less traffic) + +""" + + +class FileImport: + def __init__(self, file_path: str): + # Create Temp module + + self.device_pi_temp_module: Optional[DeviceModule] = None + self.device_probe_temp_module: Optional[DeviceModule] = None + self.device_pi_power_module: Optional[DeviceModule] = None + + self.file_path = file_path + self.data_map: list = [] + + def _set_device_module(self) -> None: + """ + If a device or device module already exists, error and fall over, we have already imported this data. + This gets around the issue of not being able to handle multiple inserts + """ + device, device_created = Device.objects.get_or_create( + name="Hub", + location="garage", + ) + + if not device_created: + raise IntegrityError("Device already exists") + + ( + self.device_pi_power_module, + pi_power_device_module_created, + ) = DeviceModule.objects.get_or_create( + device=device, + name="Device power", + location="Garage", + schema={ + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "power": { + "description": "The Hub power state", + "type": "boolean", + } + }, + "title": "Reading", + "type": "object", + }, + ) + if not pi_power_device_module_created: + raise IntegrityError( + f"Device Module {self.device_pi_power_module} already exists" + ) + + ( + self.device_pi_temp_module, + pi_temp_device_module_created, + ) = DeviceModule.objects.get_or_create( + device=device, + name="Device temperature", + location="Garage", + schema={ + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "temperature": { + "description": "The Hub device temperature", + "type": "string", + } + }, + "title": "Reading", + "type": "object", + }, + ) + if not pi_temp_device_module_created: + raise IntegrityError( + f"Device Module {self.device_pi_temp_module} already exists" + ) + + ( + self.device_probe_temp_module, + probe_device_module_created, + ) = DeviceModule.objects.get_or_create( + device=device, + name="Temperature probe", + location="Garage low", + schema={ + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "temperature": { + "description": "The temperature probe reading", + "type": "string", + } + }, + "title": "Reading", + "type": "object", + }, + ) + + if not probe_device_module_created: + raise IntegrityError( + f"Device Module {self.device_probe_temp_module} already exists" + ) + + def _process_file_line(self, line: str): + log_parts = line.split(":INFO:parent:") + # Timestamp contained some crazy characters for the power on and off logs + log_timestamp = log_parts[0].lstrip("\x00") + log_message = log_parts[1] + + # Handle started at message + if log_message.startswith("Shed pi started: "): + # Need a way to be able to record events, such as the device turning on / off + + self.data_map.append( + DeviceModuleReading( + device_module=self.device_pi_power_module, + created_at=log_timestamp, + data={"power": True}, + ) + ) + return + + elif log_message.startswith("Pi temp: "): + temps = log_message.split(": ") + + pi_temp: str = "" + probe_temp: Optional[str] = None + + if len(temps) > 2: + assert temps[0] == "Pi temp" + # Splti the next reading into 2 + partial_reading = temps[1].split(",") + assert partial_reading[1] == " probe_1 temp" + + pi_temp = partial_reading[0] + probe_temp = temps[2].strip() + else: + assert temps[0] == "Pi temp" + pi_temp = temps[1].strip() + + self.data_map.append( + DeviceModuleReading( + device_module=self.device_pi_temp_module, + created_at=log_timestamp, + data={"temperature": pi_temp}, + ) + ) + + if probe_temp: + self.data_map.append( + DeviceModuleReading( + device_module=self.device_probe_temp_module, + created_at=log_timestamp, + data={"temperature": probe_temp}, + ) + ) + + # TODO: The temp should be stored and validated as a float, Schema rule!! + # DeviceModuleReading.objects.aget_or_create( + # device_module=self.import_module, + # created_at=log_timestamp, + # data={"temperature": pi_temp}, + # ) + + print("checking for import") + + def _processed_mapped_data(self): + # TODO: Look to see if the data exists in the DB, what would this look like? + # https://gist.github.com/dorosch/6cffd2936ac05ef8794c82901ab4d6e7 + + print(f"count pre run: {DeviceModuleReading.objects.all().count()}") + DeviceModuleReading.objects.bulk_create(self.data_map, ignore_conflicts=True) + print(f"count post run: {DeviceModuleReading.objects.all().count()}") + + def _process_file(self) -> None: + with open(self.file_path, "r") as file_feed: + for log in file_feed.readlines(): + self._process_file_line(log) + + self._processed_mapped_data() + + @transaction.atomic() + def import_data(self): + # FIXME: Add timing logs to help speed this up + self._set_device_module() + + self._process_file() + + +class Command(BaseCommand): + help = "Imports historic data" + + def handle(self, *args, **options): + self.stdout.write("Started import") + + file_import = FileImport( + file_path="./historic_data_import/data/shed-pi-2024-04-02.log" + ) + file_import.import_data() + + self.stdout.write(self.style.SUCCESS("Completed import")) diff --git a/historic_data_import/migrations/__init__.py b/historic_data_import/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/historic_data_import/models.py b/historic_data_import/models.py new file mode 100644 index 0000000..6b20219 --- /dev/null +++ b/historic_data_import/models.py @@ -0,0 +1 @@ +# Create your models here. diff --git a/historic_data_import/tests.py b/historic_data_import/tests.py new file mode 100644 index 0000000..a39b155 --- /dev/null +++ b/historic_data_import/tests.py @@ -0,0 +1 @@ +# Create your tests here. diff --git a/historic_data_import/views.py b/historic_data_import/views.py new file mode 100644 index 0000000..60f00ef --- /dev/null +++ b/historic_data_import/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/shedpi_hub_dashboard/admin.py b/shedpi_hub_dashboard/admin.py index 25e6ea9..989af96 100644 --- a/shedpi_hub_dashboard/admin.py +++ b/shedpi_hub_dashboard/admin.py @@ -8,11 +8,28 @@ class DeviceAdmin(admin.ModelAdmin): list_display = ("id", "name") +class DeviceModuleReadingInlineAdmin(admin.TabularInline): + model = DeviceModuleReading + extra = 0 + can_delete = False + + # TODO: trying to limit the inline with pagination + # list_per_page = 5 # No of records per page + # + # def get_queryset(self, request): + # # TODO: return type -> QuerySet + # + # queryset = super().get_queryset(request) + # + # return queryset[:5] + + @admin.register(DeviceModule) class DeviceModuleAdmin(admin.ModelAdmin): - pass + inlines: list = [DeviceModuleReadingInlineAdmin] @admin.register(DeviceModuleReading) class DeviceModuleReadingAdmin(admin.ModelAdmin): list_display = ("id", "device_module_id", "created_at") + list_filter = ("device_module_id",) diff --git a/shedpi_hub_dashboard/migrations/0002_alter_devicemodulereading_created_at.py b/shedpi_hub_dashboard/migrations/0002_alter_devicemodulereading_created_at.py new file mode 100644 index 0000000..bb8c385 --- /dev/null +++ b/shedpi_hub_dashboard/migrations/0002_alter_devicemodulereading_created_at.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.1 on 2024-04-19 18:32 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("shedpi_hub_dashboard", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="devicemodulereading", + name="created_at", + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/shedpi_hub_dashboard/models.py b/shedpi_hub_dashboard/models.py index b2e0cff..7f9a8b9 100644 --- a/shedpi_hub_dashboard/models.py +++ b/shedpi_hub_dashboard/models.py @@ -1,6 +1,7 @@ import uuid from django.db import models +from django.utils import timezone from jsonschema import validate from shedpi_hub_dashboard.forms.fields import PrettyJsonField @@ -37,7 +38,7 @@ class DeviceModuleReading(models.Model): help_text="A device whose readings were collected.", ) data = PrettyJsonField(null=True, blank=True) - created_at = models.DateTimeField(auto_now_add=True) + created_at = models.DateTimeField(default=timezone.now) def validate_data(self) -> None: """ diff --git a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js b/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js index 26e8c7b..6c605cf 100644 --- a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js +++ b/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js @@ -73,9 +73,10 @@ section.append(tableContainer); let loadTableData = function (deviceModuleId) { // const url = section.getAttribute("data-json-feed") - const url = "http://localhost:8000//api/v1/device-module-readings/" + const url = "http://localhost:8000/api/v1/device-module-readings/" const endpoint = new URL(url); endpoint.searchParams.append("device_module", deviceModuleId); + endpoint.searchParams.append("format", "json"); // FIXME: Need data output and need headings from Schema diff --git a/shedpi_hub_dashboard/tests/test_endpoints.py b/shedpi_hub_dashboard/tests/test_endpoints.py index 0564935..82b972f 100644 --- a/shedpi_hub_dashboard/tests/test_endpoints.py +++ b/shedpi_hub_dashboard/tests/test_endpoints.py @@ -4,6 +4,7 @@ from django.urls import reverse from rest_framework import status +from shedpi_hub_dashboard.models import DeviceModuleReading from shedpi_hub_dashboard.tests.utils.factories import ( DeviceModuleFactory, DeviceModuleReadingFactory, @@ -97,3 +98,4 @@ def test_device_module_reading_submission(client): assert response.status_code == status.HTTP_201_CREATED assert response.data["data"] == data + assert DeviceModuleReading.objects.filter(device_module=device_module).count() == 1 diff --git a/shedpi_hub_example_project/settings.py b/shedpi_hub_example_project/settings.py index a0dc765..d92ec8a 100644 --- a/shedpi_hub_example_project/settings.py +++ b/shedpi_hub_example_project/settings.py @@ -38,6 +38,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "shedpi_hub_dashboard.apps.ShedpiHubDashboardConfig", + "historic_data_import.apps.HistoricDataImportConfig", "rest_framework", ] diff --git a/standalone_modules/temperature_module/temp_logger.py b/standalone_modules/temperature_module/temp_logger.py index 941e3b7..77e3940 100644 --- a/standalone_modules/temperature_module/temp_logger.py +++ b/standalone_modules/temperature_module/temp_logger.py @@ -1,3 +1,4 @@ +import json import logging import os import time @@ -5,6 +6,8 @@ from pathlib import Path from typing import Optional +import requests + """ TODO: - Separate installation script @@ -19,11 +22,34 @@ format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", ) logger = logging.getLogger("parent") + TIME_TO_SLEEP = 60 # time in seconds +HUB_ADDRESS = "http://localhost:8000" +# HUB_ADDRESS = "http://192.168.2.130:8000" + +MODULE_POWER_ID = "" +MODULE_TEMP_ID = "" +MODULE_PROBE_TEMP_ID = "" + + +def check_os(): + return os.uname()[4].startswith("arm") + + +def get_time(): + now = datetime.now(timezone.utc) + current_time = now.strftime("%H:%M:%S") # 24-Hour:Minute:Second + return current_time + class TempProbe: def __init__(self): + self.device_id: int = MODULE_PROBE_TEMP_ID + # TODO: Make this an env var, it can then be overridden by the tests + self.base_url: str = HUB_ADDRESS + self.module_reading_endpoint: str = "/api/v1/device-module-readings/" + base_dir = "/sys/bus/w1/devices/" device_folder = Path.glob(base_dir + "28*")[0] self.device_file = device_folder + "/w1_slave" @@ -50,22 +76,40 @@ def read_temp(self) -> Optional[float]: temp_c = float(temp_string) / 1000.0 return temp_c + def submit_reading(self) -> requests.Response: + """ + Submits a reading to an external endpoint -def check_os(): - return os.uname()[4].startswith("arm") + :return: + """ + probe_1_temp = self.read_temp() + logger.info(f"Submitting reading: {probe_1_temp}") -def get_cpu_temp(): - cpu_temp = os.popen("vcgencmd measure_temp").readline() + # Get a request client + # FIXME: Should this be a float or a string? Broke the test + data = {"temperature": str(probe_1_temp)} + endpoint = f"{self.base_url}{self.module_reading_endpoint}" + response = requests.post( + endpoint, + data={"device_module": self.device_id, "data": json.dumps(data)}, + ) - # Convert the temp read from the OS to a clean float - return float(cpu_temp.replace("temp=", "").replace("'C\n", "")) + # TODO: Validate the response + return response -def get_time(): - now = datetime.now(timezone.utc) - current_time = now.strftime("%H:%M:%S") # 24-Hour:Minute:Second - return current_time + +class RPIDevice: + def __init__(self): + self.device_id = MODULE_POWER_ID + self.cpu_module_id = MODULE_TEMP_ID + + def get_cpu_temp(self): + cpu_temp = os.popen("vcgencmd measure_temp").readline() + + # Convert the temp read from the OS to a clean float + return float(cpu_temp.replace("temp=", "").replace("'C\n", "")) def main(): @@ -76,11 +120,14 @@ def main(): return temp_probe = TempProbe() + rpi_device = RPIDevice() while True: - pi_temp = get_cpu_temp() - probe_1_temp = temp_probe.read_temp() - logger.info(f"Pi temp: {pi_temp}, probe_1 temp: {probe_1_temp}") + pi_temp = rpi_device.get_cpu_temp() + + logger.info(f"Pi temp: {pi_temp}") + + temp_probe.submit_reading() time.sleep(TIME_TO_SLEEP) diff --git a/standalone_modules/temperature_module/temp_readings_import.py b/standalone_modules/temperature_module/temp_readings_import.py deleted file mode 100644 index 0c87a8c..0000000 --- a/standalone_modules/temperature_module/temp_readings_import.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -A script ot import the data that has already been gathered - -Historic data structure: - "Shed pi started: {get_time()}, using version: 0.0.1" - "Pi temp: {pi_temp}, probe_1 temp: {probe_1_temp}" - -FIXME: - Allow the data submission endpoint to take multiple device modules and readings. - Example: Pi has it's own data and so does the temp probe. More effecient to send at once -""" - -data_feed = "./data/shed-pi-2024-04-02.log" - -with open(data_feed, "r") as file_feed: - - for log in file_feed.readlines(): - log_parts = log.split(":INFO:parent:") - log_timestamp = log_parts[0] - log_message = log_parts[1] - - # Handle started at message - if log_message.startswith("Shed pi started: "): - # Need a way to be able to record events, such as the device turning on / off - continue - - elif log_message.startswith("Pi temp: "): - temps = log_message.split(": ") - - if len(temps) > 2: - assert temps[0] == "Pi temp" - # Splti the next reading into 2 - partial_reading = temps[1].split(",") - assert partial_reading[1] == " probe_1 temp" - - pi_temp = partial_reading[0] - probe_temp = temps[2].strip() - else: - assert temps[0] == "Pi temp" - pi_temp = temps[1].strip() - # DeviceModuleReading.get_or_create( - # created_at= - # data= - # ) diff --git a/standalone_modules/temperature_module/test_temp_module.py b/standalone_modules/temperature_module/test_temp_module.py index c7b85c4..f930b1d 100644 --- a/standalone_modules/temperature_module/test_temp_module.py +++ b/standalone_modules/temperature_module/test_temp_module.py @@ -1,8 +1,10 @@ +import json from unittest.mock import Mock, patch import pytest -from django.urls import reverse +from rest_framework import status +from shedpi_hub_dashboard.models import DeviceModuleReading from shedpi_hub_dashboard.tests.utils.factories import ( DeviceModuleFactory, ) @@ -12,12 +14,6 @@ @patch("standalone_modules.temperature_module.temp_logger.Path") def test_temp_probe_reading_happy_path(mocked_path): - """ - TODO: - - Mock rpi - - Mock device output - - Test the case for YES from the module - """ # FIXME: Get the actual readout from the modules probe = TempProbe() probe.read_temp_raw = Mock( @@ -75,8 +71,10 @@ def test_temp_probe_reading_invalid_reading_missing_expected_params(mocked_path) probe.read_temp_raw.call_count == 2 +# Integration test, TODO: Move to Integration folder +@patch("standalone_modules.temperature_module.temp_logger.Path") @pytest.mark.django_db -def test_temperature_module_reading_submission(client): +def test_temperature_module_reading_submission(mocked_path, live_server): schema = { "$id": "https://example.com/person.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -88,14 +86,25 @@ def test_temperature_module_reading_submission(client): } device_module = DeviceModuleFactory(schema=schema) - # url = reverse("devicemodulereading-detail", kwargs={"pk": device_module.id}) - url = reverse("devicemodulereading-list") - data = {"temperature": "20.001"} - - # The below is what the module will recieve and we will be able to see that it has somehow - # response = client.post( - # url, data={"device_module": device_module.id, "data": json.dumps(data)} - # ) - # - # assert response.status_code == status.HTTP_201_CREATED - # assert response.data["data"] == data + probe = TempProbe() + probe.device_id = device_module.id + probe.base_url = live_server.url + probe.read_temp_raw = Mock( + return_value=[ + "YES", + "t=12345", + ] + ) + + response = probe.submit_reading() + + assert response.status_code == status.HTTP_201_CREATED + + response_data = json.loads(response.text) + + # assert response_data + + assert DeviceModuleReading.objects.filter(device_module=device_module).count() == 1 + + +# TODO: Test default endpoint address settings work in theory, because the test above overrides them