nim-lib/outils/communication/joplin/joplin_utils.nim

# --==--==--==--==--==--==--==--==--==--==--==--==--==--==--==-- #
# Bruno Charest 
# 2022-08-09 
# 
# __ DESCRIPTIONS __
# joplin_utils : procedure related joplin application 
#
# --==--==--==--==--==--==--==--==--==--==--==--==--==--==--==-- #
import os
import net
import times
import osproc
import jester
import std/json
import httpcore
import strutils
import strformat
import std/options
import std/httpclient
import std/asyncdispatch

import ../../commun/utils # Joplin utils procedures and types

# from os import sleep
# from posix import read, write, fdatasync, close

# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup joplin_ping data
# --==--==--==--==--==--==--==--==--==--==-- #
type
  joplin_ping* = object
    ping_result*: seq[string]
    ping_status*: bool
    req*: Request

# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup joplin_tags data
# --==--==--==--==--==--==--==--==--==--==-- #
type
  joplin_tags* = object
    id*, parent_id*, title*: seq[string]
    created_time*, updated_time*, user_created_time*, user_updated_time*,
        is_shared*: seq[int]
    req*: Request

# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup joplin_notebooks data
# --==--==--==--==--==--==--==--==--==--==-- #
type
  joplin_notebooks* = object
    id*, parent_id*, title*, share_id*, icon*: seq[string]
    created_time*, updated_time*, user_created_time*, user_updated_time*,
        is_shared*: seq[int]
    req*: Request


# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup joplin_notes data
# --==--==--==--==--==--==--==--==--==--==-- #
type
  joplin_notes* = object
    id*, parent_id*, title*, body*, author*, source_url*, source*,
        source_application*, application_data*, share_id*,
        conflict_original_id*, body_html*, base_url*: seq[string]
    updated_time*, is_conflict*, is_todo*, todo_due*,
        todo_completed*, user_created_time*, user_updated_time*,
        markup_language*, is_shared*: seq[int]
    latitude*, longitude*, altitude*, order*: seq[float]
    created_time*: seq[string]
    req*: Request

# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup joplin_note data
# --==--==--==--==--==--==--==--==--==--==-- #
type
  joplin_note* = object
    id*, parent_id*, title*, body*, author*, source_url*, source*,
        source_application*, application_data*, conflict_original_id*,
        master_key_id*, share_id*, encryption_cipher_text * : string
    created_time*, updated_time*, user_created_time*, user_updated_time*,
        is_conflict*, is_todo*, todo_due*, todo_completed*, markup_language*,
        is_shared*, encryption_applied*: int
    latitude*, longitude*, altitude*, order*: float
    req*: Request

# --==--==--==--==--==--==--==--==--==--==-- #
# TYPE : Setup process data structure
# --==--==--==--==--==--==--==--==--==--==-- #
type
  program* = object
    pid*: int
    stdout*: string
    stderr*: string

# --==--==--==--==--==--==--==--==--==--==-- #
# DURATION : duration variables
# --==--==--==--==--==--==--==--==--==--==-- #
let
  resetDuration = initDuration(seconds = 2)
  deciSecondDuration* = initDuration(milliseconds = 100)
  qtrsecondDuration* = initDuration(milliseconds = 250)

# --==--==--==--==--==--==--==--==--==--==-- #
# VAR : connexion http variables
# --==--==--==--==--==--==--==--==--==--==-- #
var
  client = newHttpClient()
  lastConnection = getTime().utc


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : reset HTTP client
# --==--==--==--==--==--==--==--==--==--==-- #
proc resetHttpClient() =
  if (getTime().utc - lastConnection) > resetDuration:
    # Probably a new timeout. We have not yet experienced a long outage.
    # We may however be entering an extended outage.
    # Creating the new clients seems to use up lots of CPU.
    # I want to do that as little as possible.
    try:
      client.close()
    except:
      echo("Attempted to close clients. Probably do not exist.")
      echo("Current exception:  ", getCurrentExceptionMsg())
    client = newHttpClient(timeout = 500)

# --==--==--==--==--==--==--==--==--==--==--==--==--==-==--==--==- #
# PROC : use the joplin API ping to validate service availibility
# --==--==--==--==--==--==--==--==--==--==--==--==--==-==--==--==- #
proc ping_joplin*(token: string): Future[joplin_ping] {.async.} =

  var j_p: joplin_ping
  var client = newAsyncHttpClient()
  var url: string
  j_p.ping_status = false
  try:
    url = fmt"http://localhost:41184/ping?token={token}"
    var json = await client.getContent(url)
    j_p.ping_result.add(json)

    echo j_p.ping_result[0]

    if j_p.ping_result[0] == "JoplinClipperServer":
      j_p.ping_status = true

  except TimeoutError, IOError, OSError:
    # I naively think I would see this thrown or the plain except below.
    # But I almost never see an Error raised.
    echo("Current Exception: ", getCurrentException().name)
    echo("Current Exception Msg: ", getCurrentExceptionMsg())
    echo("Sleeping for 1 seconds at: ", getTime().utc)
    sleep(500)
    resetHttpClient()
    j_p.ping_status = false
  except:
    echo("Current Exception: ", getCurrentException().name)
    echo("Current Exception Msg: ", getCurrentExceptionMsg())
    echo("Sleeping for 1 seconds at: ", getTime().utc)
    j_p.ping_status = false
  return j_p
    # parse jason
    #let joplin_notes_Json = parseJson(json)


  # --==--==--==--==--==--==--==--==--==--==-- #
  # PROC : get all joplin NOTEBOOKS
  # --==--==--==--==--==--==--==--==--==--==-- #
proc get_joplin_notebooks*(token: string): Future[joplin_notebooks] {.async.} =

  # Variables
  var j_nb: joplin_notebooks
  var has_more: bool = true
  var page: int = 1
  var url: string
  var client = newAsyncHttpClient()
  var pingCheck: joplin_ping

  pingCheck = waitFor ping_joplin(token)

  # check the joplin status befor query
  if pingCheck.ping_status:

    # make sure to check all pages
    while has_more == true:

      # request joplin API for notebooks
      url = fmt"http://localhost:41184/folders?page={page}&token={token}"
      echo("URL notebooks : ", url)
      var json = await client.getContent(url)

      # parse jason
      let joplin_notebooks_Json = parseJson(json)

      # valider qu'il n'y a plus de page
      if not joplin_notebooks_Json["has_more"].getBool:
        has_more = false

      # store json info into an object
      var count: int = 1
      for nb in joplin_notebooks_Json["items"]:
        j_nb.id.add(nb["id"].getstr)
        j_nb.parent_id.add(nb["parent_id"].getstr)
        j_nb.title.add(nb["title"].getstr)
        count += 1

      # aller à la page suivante
      page += 1

  return j_nb


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : get all joplin NOTES
# --==--==--==--==--==--==--==--==--==--==-- #
proc get_joplin_notes*(token: string): Future[joplin_notes] {.async.} =

  # Variables
  var j_notes: joplin_notes
  var has_more: bool = true
  var page: int = 1
  var url: string
  var client = newAsyncHttpClient()
  var pingCheck: joplin_ping
  pingCheck = waitFor ping_joplin(token)

  # check the joplin status befor query
  if pingCheck.ping_status:

    # make sure to check all pages
    while has_more == true:

      # request joplin API for Notes
      url = fmt"http://localhost:41184/notes?page={page}&fields=id,parent_id,title,created_time,updated_time&token={token}"
      echo("URL notes : ", url)
      var json = await client.getContent(url)

      # parse jason
      let joplin_notes_Json = parseJson(json)

      # valider qu'il n'y a plus de page
      if not joplin_notes_Json["has_more"].getBool:
        has_more = false

      #echo joplin_notes_Json

      # store json info into an object
      var count: int = 1
      # var epochTime: int
      # var humanTime: Time
      for nb in joplin_notes_Json["items"]:
        j_notes.id.add(nb["id"].getstr)
        j_notes.parent_id.add(nb["parent_id"].getstr)
        j_notes.title.add(nb["title"].getstr)
        #epochTime = nb["created_time"].getInt
        #humanTime = fromUnix(convert(Milliseconds, Seconds, epochTime))

        j_notes.created_time.add(convertEpochToHumanTime(nb[
            "created_time"].getInt))
        count += 1

      # aller à la page suivante
      page += 1

  return j_notes

# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : get joplin NOTE
# --==--==--==--==--==--==--==--==--==--==-- #
proc get_joplin_note*(token: string, noteid: string): Future[
    joplin_note] {.async.} =
  
  # Variables
  var j_note: joplin_note
  # var has_more: bool = true
  # var page: int = 1
  var url: string
  var client = newAsyncHttpClient()

  # Check joplin connection availibility
  var pingCheck: joplin_ping
  pingCheck = waitFor ping_joplin(token)


  # Query for a selected note
  url = fmt"http://localhost:41184/notes/{noteid}?fields=id,parent_id,title,body,created_time,updated_time,is_conflict,latitude,longitude,altitude,author,source_url,is_todo,todo_due,todo_completed,source,source_application,application_data,order,user_created_time,user_updated_time,encryption_cipher_text,encryption_applied,markup_language,is_shared,share_id,conflict_original_id,master_key_id&token={token}"
  echo("URL notes : ", url)
  var jsonNote = await client.getContent(url)

  # parse jason
  let joplin_note_Json = parseJson(jsonNote)

  # get all informations for a selected note
  j_note.id = joplin_note_Json["id"].getstr
  j_note.parent_id = joplin_note_Json["parent_id"].getstr
  j_note.title = joplin_note_Json["title"].getstr
  j_note.body = joplin_note_Json["body"].getstr
  j_note.author = joplin_note_Json["author"].getstr
  j_note.source_url = joplin_note_Json["source_url"].getstr
  j_note.source = joplin_note_Json["source"].getstr
  j_note.source_application = joplin_note_Json["source_application"].getstr
  j_note.application_data = joplin_note_Json["application_data"].getstr
  j_note.share_id = joplin_note_Json["share_id"].getstr
  j_note.conflict_original_id = joplin_note_Json["conflict_original_id"].getstr
  j_note.created_time = joplin_note_Json["created_time"].getInt
  j_note.updated_time = joplin_note_Json["updated_time"].getInt
  j_note.is_conflict = joplin_note_Json["is_conflict"].getInt
  j_note.latitude = joplin_note_Json["latitude"].getFloat
  j_note.longitude = joplin_note_Json["longitude"].getFloat
  j_note.altitude = joplin_note_Json["altitude"].getFloat
  j_note.is_todo = joplin_note_Json["is_todo"].getInt
  j_note.todo_due = joplin_note_Json["todo_due"].getInt
  j_note.todo_completed = joplin_note_Json["todo_completed"].getInt
  j_note.order = joplin_note_Json["order"].getFloat
  j_note.user_created_time = joplin_note_Json["user_created_time"].getInt
  j_note.user_updated_time = joplin_note_Json["user_updated_time"].getInt
  j_note.encryption_cipher_text = joplin_note_Json[
      "encryption_cipher_text"].getstr
  j_note.encryption_applied = joplin_note_Json["encryption_applied"].getInt
  j_note.markup_language = joplin_note_Json["markup_language"].getInt
  j_note.is_shared = joplin_note_Json["is_shared"].getInt
  j_note.master_key_id = joplin_note_Json["master_key_id"].getstr


  echo j_note
  return j_note


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : get all joplin TAGS
# --==--==--==--==--==--==--==--==--==--==-- #
proc get_joplin_tags*(token: string): Future[joplin_tags] {.async.} =

  # Variables
  var j_tags: joplin_tags
  var has_more: bool = true
  var page: int = 1
  var url: string
  var client = newAsyncHttpClient()
  var pingCheck: joplin_ping
  pingCheck = waitFor ping_joplin(token)

  # check the joplin status befor query
  if pingCheck.ping_status:

    # make sure to check all pages
    while has_more == true:

      # request joplin API for notebooks
      url = fmt"http://localhost:41184/tags?page={page}&token={token}"
      echo("URL tags : ", url)
      var json = await client.getContent(url)

      # parse jason
      let joplin_tags_Json = parseJson(json)

      # valider qu'il n'y a plus de page
      if not joplin_tags_Json["has_more"].getBool:
        has_more = false

      # store json info into an object
      var count: int = 1
      for nb in joplin_tags_Json["items"]:
        j_tags.id.add(nb["id"].getstr)
        j_tags.parent_id.add(nb["parent_id"].getstr)
        j_tags.title.add(nb["title"].getstr)
        count += 1

      # aller à la page suivante
      page += 1

  return j_tags


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : get the token from Joplin Terminal
# --==--==--==--==--==--==--==--==--==--==-- #
proc get_joplin_cli_token*(): string =
  var flagName: string = ""
  var flagValue: string = ""
  var resultExec = execCmdEx("joplin config api.token")

  if resultExec.exitCode == 0:
    let param1 = resultExec.output
    let flagSplit = param1.split(" = ")
    flagName = flagSplit[0]
    flagValue = flagSplit[1]

  return flagValue

# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : launch any program and get PID
# --==--==--==--==--==--==--==--==--==--==-- #
proc launchProgram(app: string = "", workingPath: string = "", arguments: array[
    2, string]): int {.thread.} =

  var p = startProcess(joinPath(workingPath, app), workingPath, arguments)
  let pid = p.processID()
  # var outhdl = outputHandle(p)
  # var inputhdl = inputHandle(p)
  # var errhdl = errorHandle(p)
  return pid

# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : Start joplin cli
# --==--==--==--==--==--==--==--==--==--==-- #
proc joplin_cli_start*(): int {.thread.} =

  let joplinProcessId = launchProgram("joplin", "/usr/bin/", ["server", "start"])
  return joplinProcessId


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : stop Joplin Terminal
# --==--==--==--==--==--==--==--==--==--==-- #
proc joplin_cli_stop*(): int {.thread.} =
  let joplinProcessId = launchProgram("joplin", "/usr/bin/", ["server", "stop"])
  return joplinProcessId


# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : check Joplin Terminal staus
# --==--==--==--==--==--==--==--==--==--==-- #
proc joplin_cli_status*(): bool =
  var rc = false
  var resultExec = execCmdEx("joplin server status")

  if resultExec.exitCode == 0:
    if "Server is not running" in resultExec.output:
      echo "Joplin Terminal cli status is down : ", resultExec.output
      rc = false
    else:
      echo "Joplin Terminal cli status is up : ", resultExec.output
      rc = true
  else:
    echo "Error validate joplin terminal status : ", resultExec.output
  return rc

# --==--==--==--==--==--==--==--==--==--==-- #
# PROC : start or stop Joplin Terminal
# --==--==--==--==--==--==--==--==--==--==-- #
proc joplin_cli_start_stop*(): bool =
  var idExec: int = 0
  var isStart: bool = false
  var sleep_delay_frame: int = 50
  # var sleep_max: int = 5000

  if joplin_cli_status() == false:
    idExec = joplin_cli_start()
    while joplin_cli_status() == false:
      sleep(sleep_delay_frame)
    echo "Joplin client Terminal started: ", idExec
    isStart = true

  else:
    echo "Joplin client Terminal is alredy started !"
    #if joplin_cli_status() == true:
    idExec = joplin_cli_stop()
    while joplin_cli_status() == true:
      sleep(sleep_delay_frame)
    echo "Joplin client Terminal stopped: ", idExec
    isStart = false

  return isStart




# --==--==--==--==--==--==--==--==--==--==-- #
# PROC :
# --==--==--==--==--==--==--==--==--==--==-- #

# proc get_joplin_tags_json*(token:string): Future[] {.async.} =

#   # url = fmt"http://localhost:41184/notes/{id}?fields=id,parent_id,title,body,created_time,updated_time,is_conflict,latitude,longitude,altitude,author,source_url,is_todo,todo_due,todo_completed,source,source_application,application_data,order,user_created_time,user_updated_time,encryption_cipher_text,encryption_applied,markup_language,is_shared,share_id,conflict_original_id,master_key_id&token={token}"

#   # url = fmt"http://localhost:41184/tags/{id}?fields=id,parent_id,title,created_time,updated_time,user_created_time,user_updated_time,is_shared&token={token}"

#   # http://localhost:41184/tags?token=e5f6644fbf6a97ddc55648dae72b11caecda6c6642d8ce0d3b20129b89b196385737eb908923542c3343649ebbf865b55bda031ab4c3a16edc7723ef2ad77d8f

#   # Variables
#   var j_tags: joplin_tags
#   var has_more: bool = true
#   var page: int = 1
#   var url: string
#   var client = newAsyncHttpClient()
#   var pingCheck: joplin_ping
#   pingCheck = waitFor ping_joplin(token)


#   # check the joplin status befor query
#   if pingCheck.ping_status:

#     # make sure to check all pages
#     while has_more == true:

#       # request joplin API for notebooks
#       url = fmt"http://localhost:41184/tags?page={page}&token={token}"
#       echo("URL tags : ", url)
#       var json = await client.getContent(url)

#       # parse jason
#       let joplin_tags_Json = parseJson(json)

#       # valider qu'il n'y a plus de page
#       if not joplin_tags_Json["has_more"].getBool:
#         has_more = false

#       # store json info into an object
#       var count: int = 1
#       for nb in joplin_tags_Json["items"]:
#         echo nb["id"].getstr
#         echo nb["parent_id"].getstr
#         echo nb["title"].getstr
#         count += 1

#       # aller à la page suivante
#       page += 1