## Installation of python3 venv and pip:
# sudo apt update
# sudo apt install python3-venv
# sudo apt install python3-pip
## Create and activate python virtual venv:
# python3 -m venv scrape_ven
# source scrape_venv/bin/activate
## Install required packages with pip:
# pip install bs4 paho-mqtt

import requests
from bs4 import BeautifulSoup
import json
import paho.mqtt.client as mqtt
import time

# MQTT broker and topic information
HOST = "192.168.1.5"
TOPIC = "energia/params"
USER = "mqtt"
PASSWD = "mqtt#"

# Scrape and send json to mqqt topic
def tge_to_mqtt():
    url = "https://tge.pl/energia-elektryczna-rdn"
    r = requests.get(url)

    soup = BeautifulSoup(r.content, 'lxml')
    head = soup.find('table', {'class':"footable table table-hover table-padding"})
    date = soup.find('h4', {'class':'kontrakt-date'}).text
    data = date[-10:]
    table = head.find('tbody')

    lista = []
    for row in table.find_all('tr'):
        li = []
        slown = {}
        for col in row.find_all('td'):
            li.append(col.text.replace('\n','').replace('\t',''))

        slown = {
                'Data': data,
                'Czas': li[0],
                'FIXING_I' : [{'Kurs (PLN/MWh)': li[1]},{'Wolumen (MWh)': li[2]}],
                'FIXING_II': [{'Kurs (PLN/MWh)': li[3]}, {'Wolumen (MWh)': li[4]}],
                'Notowania_ciagle': [{'Kurs (PLN/MWh)': li[5]},{'Wolumen (MWh)': li[6]}],    
            }
        lista.append(slown)

    return json.dumps(lista, indent=4)

if __name__ == "__main__":
    while True:    
    	# Create MQTT client and connect to the broker
        client = mqtt.Client()
        client.username_pw_set(USER, PASSWD)
        client.connect(HOST, 1883, 60)
        client.publish(TOPIC, payload=tge_to_mqtt())
        time.sleep(3600)								# wait 3600 sec