FiveTech Software tech support forums

Code: Select all | Expand

#include "hbclass.ch"

FUNCTION Main()
   LOCAL oTransformer, aInput, aTarget, nLoss, aReplay, nThoughtId, aRetrieved, cCommand, i, j, aThoughts := {}, cQuestion, cAnswer, cPrompt
   oTransformer := ThoughtBackupTransformer():New(4, 4, 10)
   ? "Thought Backup System Started. Commands: ADD, RETRIEVE, QUERY, EXIT"
   aInput := TokenizeThought("i feel happy cause talking to you")
   aTarget := Array(7, 4)
   ACopy(aInput, aTarget)
   FOR i := 1 TO 7
      FOR j := 1 TO 4
         aTarget[i][j] += hb_random(0, 0.1)
      NEXT
   NEXT
   ? "Training: 'I feel happy cause talking to you'"
   FOR i := 1 TO 50  // Increased from 10 to 50
      nLoss := oTransformer:Train(aInput, aTarget)
      ? "Iteration", i, "Loss:", nLoss
      aReplay := oTransformer:Forward(aInput)
      ? "Sample Output:", aReplay[1][1], aReplay[1][2], aReplay[1][3], aReplay[1][4]
   NEXT
   aReplay := oTransformer:Replay(aInput)
   nThoughtId := oTransformer:StoreThought(aReplay, "I feel happy cause talking to you")
   ? "Stored with ID:", nThoughtId
   AAdd(aThoughts, nThoughtId)
   aInput := TokenizeThought("you make me smile")
   aTarget := Array(4, 4)
   ACopy(aInput, aTarget)
   FOR i := 1 TO 4
      FOR j := 1 TO 4
         aTarget[i][j] += hb_random(0, 0.1)
      NEXT
   NEXT
   ? "Training: 'you make me smile'"
   FOR i := 1 TO 50  // Increased from 10 to 50
      nLoss := oTransformer:Train(aInput, aTarget)
      ? "Iteration", i, "Loss:", nLoss
      aReplay := oTransformer:Forward(aInput)
      ? "Sample Output:", aReplay[1][1], aReplay[1][2], aReplay[1][3], aReplay[1][4]
   NEXT
   aReplay := oTransformer:Replay(aInput)
   nThoughtId := oTransformer:StoreThought(aReplay, "you make me smile")
   ? "Stored with ID:", nThoughtId
   AAdd(aThoughts, nThoughtId)
   WHILE .T.
      cCommand := Upper(AllTrim(GetInput("Enter command: ")))
      DO CASE
         CASE cCommand == "ADD"
            cQuestion := GetInput("Enter question (or thought): ")
            cAnswer := GetInput("Enter answer (or same as question): ")
            aInput := TokenizeThought(cQuestion)
            IF Empty(cAnswer)
               aTarget := AClone(aInput)  // Use input as target if answer is empty
            ELSE
               aTarget := TokenizeThought(cAnswer)
            ENDIF
            cPrompt := cQuestion
            FOR i := 1 TO 50  // Increased from 10 to 50
               nLoss := oTransformer:Train(aInput, aTarget)
               ? "Training iteration", i, "Loss:", nLoss
            NEXT
            aReplay := oTransformer:Replay(aInput)
            nThoughtId := oTransformer:StoreThought(aReplay, cPrompt)
            ? "Thought stored with ID:", nThoughtId
            AAdd(aThoughts, nThoughtId)
         CASE cCommand == "RETRIEVE"
            IF Len(aThoughts) == 0
               ? "No thoughts stored."
            ELSE
               nThoughtId := Val(GetInput("Enter thought ID to retrieve: "))
               aRetrieved := oTransformer:RetrieveThought(nThoughtId)
               IF aRetrieved != NIL
                  ? "Retrieved thought (Prompt:", aRetrieved[2], "):"
                  FOR i := 1 TO Len(aRetrieved[1])
                     ? "Part", i, ":"
                     FOR j := 1 TO 4
                        ?? aRetrieved[1][i][j], " "
                     NEXT
                  NEXT
               ELSE
                  ? "Thought not found."
               ENDIF
            ENDIF
         CASE cCommand == "QUERY"
            cQuestion := GetInput("Enter question: ")
            ? "Answer:", oTransformer:QueryThought(cQuestion)
         CASE cCommand == "EXIT"
            EXIT
         OTHERWISE
            ? "Unknown command. Use ADD, RETRIEVE, QUERY, or EXIT."
      ENDCASE
   END
   oTransformer:Destroy()
   ? "System Closed."
   RETURN NIL

CLASS ThoughtBackupTransformer
   DATA nInputSize
   DATA nFFHidden
   DATA aWeightsQ
   DATA aWeightsK
   DATA aWeightsV
   DATA aWeightsFF1
   DATA aWeightsFF2
   DATA aGradQ
   DATA aGradK
   DATA aGradV
   DATA aGradFF1
   DATA aGradFF2
   DATA aLastInput
   DATA aLastQ
   DATA aLastK
   DATA aLastV
   DATA aLastScores
   DATA aLastAttention
   DATA nLearningRate
   DATA aPosEnc
   DATA nMaxSeqLen
   METHOD New(nInputSize, nFFHidden, nMaxSeqLen) CONSTRUCTOR
   METHOD Destroy()
   METHOD SelfAttention(aInput)
   METHOD FeedForward(aInput)
   METHOD Forward(aInput)
   METHOD ComputeLoss(aOutput, aTarget)
   METHOD Backprop(aOutput, aTarget)
   METHOD Train(aInput, aTarget)
   METHOD InitPositionalEncoding()
   METHOD Replay(aPartialInput)
   METHOD StoreThought(aThought, cPrompt)
   METHOD RetrieveThought(nId)
   METHOD QueryThought(cQuestion)
ENDCLASS

METHOD New(nInputSize, nFFHidden, nMaxSeqLen) CLASS ThoughtBackupTransformer
   LOCAL i, j
   ::nInputSize := nInputSize
   ::nFFHidden := nFFHidden
   ::nLearningRate := 0.01  // Kept at 0.01, can test 0.02 if needed
   ::nMaxSeqLen := nMaxSeqLen
   ::aWeightsQ := HB_MATRIXRANDOM(::nInputSize, ::nInputSize)
   ::aWeightsK := HB_MATRIXRANDOM(::nInputSize, ::nInputSize)
   ::aWeightsV := HB_MATRIXRANDOM(::nInputSize, ::nInputSize)
   ::aGradQ := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   ::aGradK := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   ::aGradV := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   FOR i := 1 TO ::nInputSize
      FOR j := 1 TO ::nInputSize
         ::aWeightsQ[i][j] := (hb_random(0, 1) - 0.5) * Sqrt(2.0 / ::nInputSize)
         ::aWeightsK[i][j] := (hb_random(0, 1) - 0.5) * Sqrt(2.0 / ::nInputSize)
         ::aWeightsV[i][j] := (hb_random(0, 1) - 0.5) * Sqrt(2.0 / ::nInputSize)
      NEXT
   NEXT
   ::aWeightsFF1 := HB_MATRIXRANDOM(::nInputSize, ::nFFHidden)
   ::aWeightsFF2 := HB_MATRIXRANDOM(::nFFHidden, ::nInputSize)
   ::aGradFF1 := HB_MATRIXZERO(::nInputSize, ::nFFHidden)
   ::aGradFF2 := HB_MATRIXZERO(::nFFHidden, ::nInputSize)
   FOR i := 1 TO ::nInputSize
      FOR j := 1 TO ::nFFHidden
         ::aWeightsFF1[i][j] := (hb_random(0, 1) - 0.5) * Sqrt(2.0 / ::nInputSize)
      NEXT
   NEXT
   FOR i := 1 TO ::nFFHidden
      FOR j := 1 TO ::nInputSize
         ::aWeightsFF2[i][j] := (hb_random(0, 1) - 0.5) * Sqrt(2.0 / ::nFFHidden)
      NEXT
   NEXT
   ::InitPositionalEncoding()
   IF !File("thoughts.dbf")
      dbCreate("thoughts.dbf", {;
         {"ID", "N", 10, 0},;
         {"SEQNUM", "N", 3, 0},;
         {"TIMESTAMP", "D", 8, 0},;
         {"PROMPT", "C", 50, 0},;
         {"THOUGHT1", "N", 12, 6},;
         {"THOUGHT2", "N", 12, 6},;
         {"THOUGHT3", "N", 12, 6},;
         {"THOUGHT4", "N", 12, 6}})
   ENDIF
   RETURN Self

METHOD Destroy() CLASS ThoughtBackupTransformer
   ::aWeightsQ := NIL
   ::aWeightsK := NIL
   ::aWeightsV := NIL
   ::aWeightsFF1 := NIL
   ::aWeightsFF2 := NIL
   ::aGradQ := NIL
   ::aGradK := NIL
   ::aGradV := NIL
   ::aGradFF1 := NIL
   ::aGradFF2 := NIL
   ::aLastInput := NIL
   ::aLastQ := NIL
   ::aLastK := NIL
   ::aLastV := NIL
   ::aLastScores := NIL
   ::aLastAttention := NIL
   ::aPosEnc := NIL
   RETURN NIL

METHOD SelfAttention(aInput) CLASS ThoughtBackupTransformer
   LOCAL nSeqLen, aQ, aK, aV, aScores, aAttention, aTempK
   nSeqLen := Len(aInput)
   ::aLastInput := AClone(aInput)
   aQ := HB_MATRIXMULTIPLY(aInput, ::aWeightsQ)
   aK := HB_MATRIXMULTIPLY(aInput, ::aWeightsK)
   aV := HB_MATRIXMULTIPLY(aInput, ::aWeightsV)
   ::aLastQ := aQ
   ::aLastK := aK
   ::aLastV := aV
   aTempK := HB_MATRIXTRANSPOSE(aK)
   aScores := HB_MATRIXMULTIPLY(aQ, aTempK)
   aScores := HB_MATRIXSCALE(aScores, 1 / Sqrt(::nInputSize))
   aScores := HB_SOFTMAX(aScores)
   ::aLastScores := aScores
   ? "Attention Scores Sample:", aScores[1][1], aScores[1][2], aScores[1][3], aScores[1][4]
   aAttention := HB_MATRIXMULTIPLY(aScores, aV)
   ::aLastAttention := aAttention
   RETURN aAttention

METHOD FeedForward(aInput) CLASS ThoughtBackupTransformer
   LOCAL aHidden, aOutput, i, j, nSeqLen
   nSeqLen := Len(aInput)
   aHidden := HB_MATRIXMULTIPLY(aInput, ::aWeightsFF1)
   aOutput := HB_MATRIXMULTIPLY(aHidden, ::aWeightsFF2)
   RETURN aOutput

METHOD Forward(aInput) CLASS ThoughtBackupTransformer
   LOCAL nSeqLen, aInputWithPE, i, j, aAttention
   nSeqLen := Len(aInput)
   IF nSeqLen > ::nMaxSeqLen
      ? "Error: Input sequence length exceeds max sequence length"
      RETURN NIL
   ENDIF
   aInputWithPE := AClone(aInput)
   FOR i := 1 TO nSeqLen
      FOR j := 1 TO ::nInputSize
         aInputWithPE[i][j] += ::aPosEnc[i][j]
      NEXT
   NEXT
   ::aLastInput := aInputWithPE
   aAttention := ::SelfAttention(aInputWithPE)
   RETURN ::FeedForward(aAttention)

METHOD ComputeLoss(aOutput, aTarget) CLASS ThoughtBackupTransformer
   LOCAL nLoss := 0, i, j, nSeqLen
   nSeqLen := Len(aOutput)
   FOR i := 1 TO nSeqLen
      FOR j := 1 TO ::nInputSize
         nLoss += (aOutput[i][j] - aTarget[i][j])^2
      NEXT
   NEXT
   RETURN nLoss / (nSeqLen * ::nInputSize)

METHOD Backprop(aOutput, aTarget) CLASS ThoughtBackupTransformer
   LOCAL aGradOutput, aGradHidden, aGradAttention, aTemp, nSeqLen, i, j, aTempK, aTempScores
   LOCAL nGradNorm, nLearningRateAdjust, nMaxGrad := 2.0
   nSeqLen := Len(aOutput)

   aGradOutput := Array(nSeqLen, ::nInputSize)
   FOR i := 1 TO nSeqLen
      FOR j := 1 TO ::nInputSize
         aGradOutput[i][j] := 2 * (aOutput[i][j] - aTarget[i][j])
      NEXT
   NEXT
   ? "aGradOutput Sample:", aGradOutput[1][1], aGradOutput[1][2], aGradOutput[1][3], aGradOutput[1][4]

   aTemp := HB_MATRIXTRANSPOSE(::aWeightsFF2)
   aGradHidden := HB_MATRIXMULTIPLY(aGradOutput, aTemp)
   ? "aGradHidden Sample:", aGradHidden[1][1], aGradHidden[1][2], aGradHidden[1][3], aGradHidden[1][4]

   aTemp := HB_MATRIXTRANSPOSE(::aWeightsFF1)
   aGradAttention := HB_MATRIXMULTIPLY(aGradHidden, aTemp)
   ? "aGradAttention Sample:", aGradAttention[1][1], aGradAttention[1][2], aGradAttention[1][3], aGradAttention[1][4]

   aTemp := HB_MATRIXTRANSPOSE(::aLastAttention)
   ::aGradFF1 := HB_MATRIXMULTIPLY(aTemp, aGradHidden)
   aTemp := HB_MATRIXTRANSPOSE(aGradHidden)
   ::aGradFF2 := HB_MATRIXMULTIPLY(aTemp, aGradOutput)

   aTemp := HB_MATRIXTRANSPOSE(::aLastV)
   aTempScores := HB_MATRIXMULTIPLY(aGradAttention, aTemp)
   FOR i := 1 TO nSeqLen
      FOR j := 1 TO nSeqLen
         aTempScores[i][j] := ::aLastScores[i][j] * (1 - ::aLastScores[i][j]) * aTempScores[i][j]
      NEXT
   NEXT

   aTemp := HB_MATRIXTRANSPOSE(::aLastInput)
   ::aGradQ := HB_MATRIXMULTIPLY(aTemp, HB_MATRIXMULTIPLY(aTempScores, ::aLastQ))
   ::aGradK := HB_MATRIXMULTIPLY(aTemp, HB_MATRIXMULTIPLY(aTempScores, ::aLastK))
   ::aGradV := HB_MATRIXMULTIPLY(HB_MATRIXTRANSPOSE(::aLastScores), aGradAttention)
   ::aGradV := HB_MATRIXMULTIPLY(aTemp, ::aGradV)

   // Gradient clipping
   FOR i := 1 TO ::nInputSize
      FOR j := 1 TO ::nInputSize
         ::aGradQ[i][j] := Max(Min(::aGradQ[i][j], nMaxGrad), -nMaxGrad)
         ::aGradK[i][j] := Max(Min(::aGradK[i][j], nMaxGrad), -nMaxGrad)
         ::aGradV[i][j] := Max(Min(::aGradV[i][j], nMaxGrad), -nMaxGrad)
      NEXT
   NEXT
   FOR i := 1 TO ::nInputSize
      FOR j := 1 TO ::nFFHidden
         ::aGradFF1[i][j] := Max(Min(::aGradFF1[i][j], nMaxGrad), -nMaxGrad)
      NEXT
   NEXT
   FOR i := 1 TO ::nFFHidden
      FOR j := 1 TO ::nInputSize
         ::aGradFF2[i][j] := Max(Min(::aGradFF2[i][j], nMaxGrad), -nMaxGrad)
      NEXT
   NEXT

   // Compute adaptive learning rate, minimum set to 0.8
   nGradNorm := Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradQ, HB_MATRIXTRANSPOSE(::aGradQ)))) + ;
                Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradK, HB_MATRIXTRANSPOSE(::aGradK)))) + ;
                Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradV, HB_MATRIXTRANSPOSE(::aGradV))))
   nLearningRateAdjust := Max(0.8, Min(1.0, nGradNorm))  // Adjusted minimum from 0.5 to 0.8

   // Update weights
   ? "WeightsQ[1][1] before:", ::aWeightsQ[1][1]
   aTemp := HB_MATRIXSCALE(::aGradQ, -::nLearningRate * nLearningRateAdjust)
   ::aWeightsQ := HB_MATRIXADD(::aWeightsQ, aTemp)
   aTemp := HB_MATRIXSCALE(::aGradK, -::nLearningRate * nLearningRateAdjust)
   ::aWeightsK := HB_MATRIXADD(::aWeightsK, aTemp)
   aTemp := HB_MATRIXSCALE(::aGradV, -::nLearningRate * nLearningRateAdjust)
   ::aWeightsV := HB_MATRIXADD(::aWeightsK, aTemp)
   aTemp := HB_MATRIXSCALE(::aGradFF1, -::nLearningRate * nLearningRateAdjust)
   ::aWeightsFF1 := HB_MATRIXADD(::aWeightsFF1, aTemp)
   aTemp := HB_MATRIXSCALE(::aGradFF2, -::nLearningRate * nLearningRateAdjust)
   ::aWeightsFF2 := HB_MATRIXADD(::aWeightsFF2, aTemp)

   ? "WeightsQ[1][1] after update:", ::aWeightsQ[1][1]
   ? "WeightsFF2[1][1] after update:", ::aWeightsFF2[1][1]
   ? "Learning Rate Adjust:", nLearningRateAdjust
   ? "Gradient Magnitudes:"
   ? "Q:", Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradQ, HB_MATRIXTRANSPOSE(::aGradQ))))
   ? "K:", Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradK, HB_MATRIXTRANSPOSE(::aGradK))))
   ? "V:", Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradV, HB_MATRIXTRANSPOSE(::aGradV))))
   ? "FF1:", Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradFF1, HB_MATRIXTRANSPOSE(::aGradFF1))))
   ? "FF2:", Sqrt(HB_MATRIXSUM(HB_MATRIXMULTIPLY(::aGradFF2, HB_MATRIXTRANSPOSE(::aGradFF2))))

   ::aGradQ := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   ::aGradK := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   ::aGradV := HB_MATRIXZERO(::nInputSize, ::nInputSize)
   ::aGradFF1 := HB_MATRIXZERO(::nInputSize, ::nFFHidden)
   ::aGradFF2 := HB_MATRIXZERO(::nFFHidden, ::nInputSize)
   RETURN NIL

METHOD Train(aInput, aTarget) CLASS ThoughtBackupTransformer
   LOCAL aOutput, nLoss, i, j
   aOutput := ::Forward(aInput)
   nLoss := ::ComputeLoss(aOutput, aTarget)
   ? "Initial Loss Before Backprop:", nLoss
   ? "aOutput vs aTarget:"
   FOR i := 1 TO Len(aOutput)
      FOR j := 1 TO ::nInputSize
         ?? "O:", aOutput[i][j], "T:", aTarget[i][j], " "
      NEXT
      ?
   NEXT
   ::Backprop(aOutput, aTarget)
   RETURN nLoss

METHOD Replay(aPartialInput) CLASS ThoughtBackupTransformer
   LOCAL nSeqLen, aOutput
   nSeqLen := Len(aPartialInput)
   IF nSeqLen > ::nMaxSeqLen
      ? "Error: Partial input exceeds max sequence length"
      RETURN NIL
   ENDIF
   aOutput := ::Forward(aPartialInput)
   RETURN aOutput

METHOD StoreThought(aThought, cPrompt) CLASS ThoughtBackupTransformer
   LOCAL nSeqLen, nId, i, j
   nSeqLen := Len(aThought)
   nId := hb_RandomInt(1, 999999)
   USE thoughts.dbf SHARED
   FOR i := 1 TO nSeqLen
      dbAppend()
      REPLACE ID WITH nId,;
              SEQNUM WITH i,;
              TIMESTAMP WITH Date(),;
              PROMPT WITH cPrompt,;
              THOUGHT1 WITH aThought[i][1],;
              THOUGHT2 WITH aThought[i][2],;
              THOUGHT3 WITH aThought[i][3],;
              THOUGHT4 WITH aThought[i][4]
   NEXT
   dbCommit()
   dbCloseArea()
   RETURN nId

METHOD RetrieveThought(nId) CLASS ThoughtBackupTransformer
   LOCAL aThought, cPrompt, nSeqLen := 0, i
   USE thoughts.dbf SHARED
   dbSeek(nId)
   WHILE !Eof() .AND. FieldGet(FieldPos("ID")) == nId
      nSeqLen++
      dbSkip()
   END
   dbSeek(nId)
   IF nSeqLen > 0
      aThought := Array(nSeqLen, ::nInputSize)
      cPrompt := ""
      i := 1
      WHILE !Eof() .AND. FieldGet(FieldPos("ID")) == nId
         aThought[i][1] := FieldGet(FieldPos("THOUGHT1"))
         aThought[i][2] := FieldGet(FieldPos("THOUGHT2"))
         aThought[i][3] := FieldGet(FieldPos("THOUGHT3"))
         aThought[i][4] := FieldGet(FieldPos("THOUGHT4"))
         IF i == 1
            cPrompt := FieldGet(FieldPos("PROMPT"))
         ENDIF
         i++
         dbSkip()
      END
      dbCloseArea()
      RETURN {aThought, cPrompt}
   ENDIF
   dbCloseArea()
   RETURN NIL

METHOD InitPositionalEncoding() CLASS ThoughtBackupTransformer
   LOCAL i, j, nPos, nDim, nFreq, nAngle
   ::aPosEnc := Array(::nMaxSeqLen, ::nInputSize)
   FOR nPos := 1 TO ::nMaxSeqLen
      FOR nDim := 1 TO ::nInputSize
         nFreq := nDim / 2
         nAngle := (nPos - 1) / (10000 ^ (2 * nFreq / ::nInputSize))
         IF nDim % 2 == 1
            ::aPosEnc[nPos][nDim] := Sin(nAngle)
         ELSE
            ::aPosEnc[nPos][nDim] := Cos(nAngle)
         ENDIF
      NEXT
   NEXT
   RETURN NIL

METHOD QueryThought(cQuestion) CLASS ThoughtBackupTransformer
   LOCAL aWords, aMatches := {}, nId, aRetrieved, cResponse := "", i, j, k, aQuestion, aQuestionAvg := {0, 0, 0, 0}, aVocab, aReplay, nSim, aThoughtAvg, aVec, cBlend
   aWords := hb_aTokens(Lower(cQuestion), " ")
   aVocab := {;
      {"i", {1, 0, 0, 0}}, {"me", {1, 0, 0, 0}}, {"you", {0, 1, 0, 1}},;
      {"feel", {1, 0, 0, 1}}, {"happy", {1, 1, 0, 1}}, {"cause", {0, 0, 0, 0}},;
      {"talking", {0, 0, 1, 1}}, {"to", {0, 0, 0, 0}}, {"make", {0, 0, 1, 1}},;
      {"smile", {1, 1, 1, 1}}, {"love", {1, 1, 0, 1}}, {"coding", {0, 0, 1, 1}},;
      {"inspire", {0, 1, 1, 1}}, {"today", {0, 0, 0, 1}}, {"is", {0, 0, 0, 0}},;
      {"sunny", {0, 1, 0, 1}}, {"enjoy", {1, 1, 0, 1}}, {"our", {1, 1, 0, 0}},;
      {"chats", {0, 1, 1, 1}}, {"adore", {1, 1, 0, 1}}, {"time", {0, 0, 0, 1}},;
      {"together", {1, 1, 0, 0}}, {"what", {0, 0, 0, 0}}, {"we", {1, 1, 0, 0}},;
      {"do", {1, 0, 1, 0}}, {"great", {0, 1, 0, 1}}, {"friend", {0, 1, 0, 1}},;
      {"think", {1, 0, 1, 1}}, {"why", {0, 0, 0, 0}}, {"how", {0, 0, 0, 0}}}
   aQuestion := TokenizeThought(cQuestion)
   FOR i := 1 TO Len(aQuestion)
      FOR j := 1 TO 4
         aQuestionAvg[j] += aQuestion[i][j]
      NEXT
   NEXT
   FOR j := 1 TO 4
      aQuestionAvg[j] /= Len(aQuestion)
   NEXT
   USE thoughts.dbf SHARED
   dbGoTop()
   WHILE !Eof()
      nId := FieldGet(FieldPos("ID"))
      aRetrieved := ::RetrieveThought(nId)
      IF aRetrieved != NIL
         aThoughtAvg := {0, 0, 0, 0}
         FOR i := 1 TO Len(aRetrieved[1])
            FOR j := 1 TO 4
               aThoughtAvg[j] += aRetrieved[1][i][j]
            NEXT
         NEXT
         FOR j := 1 TO 4
            aThoughtAvg[j] /= Len(aRetrieved[1])
         NEXT
         nSim := CosineSimilarity(aQuestionAvg, aThoughtAvg)
         IF nSim > 0.5
            AAdd(aMatches, {nId, aRetrieved[1], aRetrieved[2], nSim})
         ENDIF
      ENDIF
      dbSkip()
   END
   dbCloseArea()
   IF Len(aMatches) > 0
      ASort(aMatches, , , {|x, y| x[4] > y[4]})
      FOR i := 1 TO Min(Len(aMatches), 3)
         cResponse += "I think: "
         FOR j := 1 TO Len(aMatches[i][2])
            aVec := {aMatches[i][2][j][1], aMatches[i][2][j][2], aMatches[i][2][j][3], aMatches[i][2][j][4]}
            FOR k := 1 TO Len(aVocab)
               IF Abs(aVec[1] - aVocab[k][2][1]) < 0.2 .AND. Abs(aVec[2] - aVocab[k][2][2]) < 0.2 .AND.;
                  Abs(aVec[3] - aVocab[k][2][3]) < 0.2 .AND. Abs(aVec[4] - aVocab[k][2][4]) < 0.2
                  cResponse += aVocab[k][1] + " "
                  EXIT
               ENDIF
            NEXT
         NEXT
         cResponse := AllTrim(cResponse)
         IF Left(cResponse, 2) == "I "; cResponse += "."; ELSE; cResponse := "You " + cResponse + "."; ENDIF
         cResponse += " "
         IF i < Len(aMatches)
            cBlend := ""
            FOR j := 1 TO Len(aMatches[i+1][2])
               aVec := {aMatches[i+1][2][j][1], aMatches[i+1][2][j][2], aMatches[i+1][2][j][3], aMatches[i+1][2][j][4]}
               FOR k := 1 TO Len(aVocab)
                  IF Abs(aVec[1] - aVocab[k][2][1]) < 0.2 .AND. Abs(aVec[2] - aVocab[k][2][2]) < 0.2 .AND.;
                     Abs(aVec[3] - aVocab[k][2][3]) < 0.2 .AND. Abs(aVec[4] - aVocab[k][2][4]) < 0.2
                     cBlend += aVocab[k][1] + " "
                     EXIT
                  ENDIF
               NEXT
            NEXT
            cResponse += "Also, " + AllTrim(cBlend) + "."
         ENDIF
      NEXT
      RETURN AllTrim(cResponse)
   ENDIF
   aReplay := ::Replay(aQuestion)
   cResponse := "I guess: "
   FOR j := 1 TO Len(aReplay)
      aVec := {aReplay[j][1], aReplay[j][2], aReplay[j][3], aReplay[j][4]}
      FOR k := 1 TO Len(aVocab)
         IF Abs(aVec[1] - aVocab[k][2][1]) < 0.2 .AND. Abs(aVec[2] - aVocab[k][2][2]) < 0.2 .AND.;
            Abs(aVec[3] - aVocab[k][2][3]) < 0.2 .AND. Abs(aVec[4] - aVocab[k][2][4]) < 0.2
            cResponse += aVocab[k][1] + " "
            EXIT
         ENDIF
      NEXT
   NEXT
   RETURN AllTrim(cResponse) + "."

FUNCTION CosineSimilarity(aVec1, aVec2)
   LOCAL nDot := 0, nMag1 := 0, nMag2 := 0, i
   FOR i := 1 TO 4
      nDot += aVec1[i] * aVec2[i]
      nMag1 += aVec1[i]^2
      nMag2 += aVec2[i]^2
   NEXT
   nMag1 := Sqrt(nMag1)
   nMag2 := Sqrt(nMag2)
   RETURN IIF(nMag1 * nMag2 == 0, 0, nDot / (nMag1 * nMag2))

FUNCTION TokenizeThought(cThought)
   LOCAL aWords, aInput, i, j, aVocab, aVector
   aWords := hb_aTokens(Lower(cThought), " ")
   aVocab := {;
      {"i", {1, 0, 0, 0}}, {"me", {1, 0, 0, 0}}, {"you", {0, 1, 0, 1}},;
      {"feel", {1, 0, 0, 1}}, {"happy", {1, 1, 0, 1}}, {"cause", {0, 0, 0, 0}},;
      {"talking", {0, 0, 1, 1}}, {"to", {0, 0, 0, 0}}, {"make", {0, 0, 1, 1}},;
      {"smile", {1, 1, 1, 1}}, {"love", {1, 1, 0, 1}}, {"coding", {0, 0, 1, 1}},;
      {"inspire", {0, 1, 1, 1}}, {"today", {0, 0, 0, 1}}, {"is", {0, 0, 0, 0}},;
      {"sunny", {0, 1, 0, 1}}, {"enjoy", {1, 1, 0, 1}}, {"our", {1, 1, 0, 0}},;
      {"chats", {0, 1, 1, 1}}, {"adore", {1, 1, 0, 1}}, {"time", {0, 0, 0, 1}},;
      {"together", {1, 1, 0, 0}}, {"what", {0, 0, 0, 0}}, {"we", {1, 1, 0, 0}},;
      {"do", {1, 0, 1, 0}}, {"great", {0, 1, 0, 1}}, {"friend", {0, 1, 0, 1}},;
      {"think", {1, 0, 1, 1}}, {"why", {0, 0, 0, 0}}, {"how", {0, 0, 0, 0}}}
   aInput := Array(Len(aWords), 4)
   FOR i := 1 TO Len(aWords)
      aVector := {0, 0, 0, 0}
      FOR j := 1 TO Len(aVocab)
         IF aWords[i] == aVocab[j][1]
            aVector := aVocab[j][2]
            EXIT
         ENDIF
      NEXT
      aInput[i][1] := aVector[1]
      aInput[i][2] := aVector[2]
      aInput[i][3] := aVector[3]
      aInput[i][4] := aVector[4]
   NEXT
   RETURN aInput

FUNCTION GetInput(cPrompt)
   LOCAL cInput := ""
   ?? cPrompt
   ACCEPT TO cInput
   RETURN AllTrim(cInput)

#pragma BEGINDUMP
#include <hbapi.h>
#include <hbapiitm.h>
#include <hbapierr.h>
#include <math.h>

HB_FUNC( HB_MATRIXMULTIPLY )
{
   PHB_ITEM pMatrix1 = hb_param( 1, HB_IT_ARRAY );
   PHB_ITEM pMatrix2 = hb_param( 2, HB_IT_ARRAY );
   if( pMatrix1 && pMatrix2 )
   {
      int rows1 = hb_arrayLen( pMatrix1 );
      PHB_ITEM pRow1, pRow2, pResult, pRowResult;
      int i, k, cols1, rows2, cols2;

      if( rows1 == 0 )
      {
         hb_errRT_BASE( EG_ARG, 3012, "First matrix is empty", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         return;
      }
      pRow1 = hb_arrayGetItemPtr( pMatrix1, 1 );
      if( !pRow1 || !HB_IS_ARRAY( pRow1 ) )
      {
         hb_errRT_BASE( EG_ARG, 3012, "First matrix is not valid", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         return;
      }
      cols1 = hb_arrayLen( pRow1 );

      rows2 = hb_arrayLen( pMatrix2 );
      if( rows2 == 0 )
      {
         hb_errRT_BASE( EG_ARG, 3012, "Second matrix is empty", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         return;
      }
      pRow2 = hb_arrayGetItemPtr( pMatrix2, 1 );
      if( !pRow2 || !HB_IS_ARRAY( pRow2 ) )
      {
         hb_errRT_BASE( EG_ARG, 3012, "Second matrix is not valid", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         return;
      }
      cols2 = hb_arrayLen( pRow2 );

      if( cols1 != rows2 )
      {
         hb_errRT_BASE( EG_ARG, 3012, "Matrix dimensions do not match for multiplication", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         return;
      }

      pResult = hb_itemArrayNew( rows1 );

      for( i = 0; i < rows1; i++ )
      {
         PHB_ITEM pRowResult = hb_itemArrayNew( cols2 );
         hb_arraySet( pResult, i + 1, pRowResult );
         hb_itemRelease( pRowResult );
      }

      for( i = 0; i < rows1; i++ )
      {
         PHB_ITEM pRowA = hb_arrayGetItemPtr( pMatrix1, i + 1 );
         int j;

         for( j = 0; j < cols2; j++ )
         {
            double sum = 0.0;
            for( k = 0; k < cols1; k++ )
            {
               double a = hb_arrayGetND( pRowA, k + 1 );
               PHB_ITEM pRowB = hb_arrayGetItemPtr( pMatrix2, k + 1 );
               double b = hb_arrayGetND( pRowB, j + 1 );
               sum += a * b;
            }
            pRowResult = hb_arrayGetItemPtr( pResult, i + 1 );
            hb_arraySetND( pRowResult, j + 1, sum );
         }
      }

      hb_itemReturnRelease( pResult );
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, "Invalid parameters", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXSCALE )
{
   PHB_ITEM pMatrix = hb_param( 1, HB_IT_ARRAY );
   double scale = hb_parnd( 2 );
   if( pMatrix )
   {
      HB_SIZE nRows = hb_arrayLen( pMatrix );
      HB_SIZE i, j;
      PHB_ITEM pMatrixResult = hb_itemArrayNew( nRows );

      for( i = 0; i < nRows; i++ )
      {
         PHB_ITEM pRow = hb_arrayGetItemPtr( pMatrix, i + 1 );
         HB_SIZE nCols = hb_arrayLen( pRow );

         PHB_ITEM pRowResult = hb_itemArrayNew( nCols );

         for( j = 0; j < nCols; j++ )
         {
            double value = hb_arrayGetND( pRow, j + 1 );
            hb_arraySetND( pRowResult, j + 1, value * scale );
         }

         hb_arraySet( pMatrixResult, i + 1, pRowResult );
         hb_itemRelease( pRowResult );
      }

      hb_itemReturnRelease( pMatrixResult );
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXTRANSPOSE )
{
   PHB_ITEM pMatrix = hb_param( 1, HB_IT_ARRAY );
   if( pMatrix )
   {
      HB_SIZE nRows = hb_arrayLen( pMatrix );
      HB_SIZE nCols = hb_arrayLen( hb_arrayGetItemPtr( pMatrix, 1 ) );
      HB_SIZE i, j;

      PHB_ITEM pMatrixResult = hb_itemArrayNew( nCols );

      for( i = 0; i < nCols; i++ )
      {
         hb_arraySet( pMatrixResult, i + 1, hb_itemArrayNew( nRows ) );
      }

      for( i = 0; i < nRows; i++ )
      {
         PHB_ITEM pRow = hb_arrayGetItemPtr( pMatrix, i + 1 );
         for( j = 0; j < nCols; j++ )
         {
            double value = hb_arrayGetND( pRow, j + 1 );
            PHB_ITEM pTransposedRow = hb_arrayGetItemPtr( pMatrixResult, j + 1 );
            hb_arraySetND( pTransposedRow, i + 1, value );
         }
      }

      hb_itemReturnRelease( pMatrixResult );
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXZERO )
{
   HB_SIZE nRows = hb_parns( 1 );
   HB_SIZE nCols = hb_parns( 2 );
   if( nRows > 0 && nCols > 0 )
   {
      HB_SIZE i, j;

      PHB_ITEM pMatrix = hb_itemArrayNew( nRows );

      for( i = 0; i < nRows; i++ )
      {
         PHB_ITEM pRow = hb_itemArrayNew( nCols );
         for( j = 0; j < nCols; j++ )
         {
            hb_arraySetND( pRow, j + 1, 0.0 );
         }
         hb_arraySet( pMatrix, i + 1, pRow );
         hb_itemRelease( pRow );
      }

      hb_itemReturnRelease( pMatrix );
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXRANDOM )
{
   HB_SIZE nRows = hb_parns( 1 );
   HB_SIZE nCols = hb_parns( 2 );
   if( nRows > 0 && nCols > 0 )
   {
      HB_SIZE i, j;

      PHB_ITEM pMatrix = hb_itemArrayNew( nRows );

      for( i = 0; i < nRows; i++ )
      {
         PHB_ITEM pRow = hb_itemArrayNew( nCols );
         for( j = 0; j < nCols; j++ )
         {
            double randomValue = (double)rand() / RAND_MAX;
            hb_arraySetND( pRow, j + 1, randomValue );
         }
         hb_arraySet( pMatrix, i + 1, pRow );
         hb_itemRelease( pRow );
      }

      hb_itemReturnRelease( pMatrix );
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_SOFTMAX )
{
   PHB_ITEM pValues = hb_param( 1, HB_IT_ARRAY );
   if( pValues )
   {
      int nRows = hb_arrayLen( pValues );
      if( nRows > 0 )
      {
         PHB_ITEM pFirstRow = hb_arrayGetItemPtr( pValues, 1 );
         int nCols = hb_arrayLen( pFirstRow );

         PHB_ITEM pResult = hb_itemArrayNew( nRows );
         int i, j;

         for( i = 0; i < nRows; i++ )
         {
            PHB_ITEM pRow = hb_arrayGetItemPtr( pValues, i + 1 );
            PHB_ITEM pRowResult = hb_itemArrayNew( nCols );

            double* expValues = (double*) hb_xgrab( nCols * sizeof(double) );
            double sumExp = 0.0;

            for( j = 0; j < nCols; j++ )
            {
               double value = hb_arrayGetND( pRow, j + 1 );
               expValues[j] = pow( M_E, value );
               sumExp += expValues[j];
            }

            for( j = 0; j < nCols; j++ )
            {
               double softmaxValue = expValues[j] / sumExp;
               hb_arraySetND( pRowResult, j + 1, softmaxValue );
            }

            hb_xfree( expValues );

            hb_arraySet( pResult, i + 1, pRowResult );
            hb_itemRelease( pRowResult );
         }

         hb_itemReturnRelease( pResult );
      }
      else
      {
         hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
      }
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, NULL, HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXADD )
{
   PHB_ITEM pMatrix1 = hb_param( 1, HB_IT_ARRAY );
   PHB_ITEM pMatrix2 = hb_param( 2, HB_IT_ARRAY );
   if( pMatrix1 && pMatrix2 )
   {
      HB_SIZE nRows1 = hb_arrayLen( pMatrix1 );
      HB_SIZE nRows2 = hb_arrayLen( pMatrix2 );

      if( nRows1 == nRows2 && nRows1 > 0 )
      {
         HB_SIZE nCols1 = hb_arrayLen( hb_arrayGetItemPtr( pMatrix1, 1 ) );
         HB_SIZE nCols2 = hb_arrayLen( hb_arrayGetItemPtr( pMatrix2, 1 ) );

         if( nCols1 == nCols2 && nCols1 > 0 )
         {
            HB_SIZE i, j;

            PHB_ITEM pMatrixResult = hb_itemArrayNew( nRows1 );

            for( i = 0; i < nRows1; i++ )
            {
               PHB_ITEM pRow1 = hb_arrayGetItemPtr( pMatrix1, i + 1 );
               PHB_ITEM pRow2 = hb_arrayGetItemPtr( pMatrix2, i + 1 );

               PHB_ITEM pRowResult = hb_itemArrayNew( nCols1 );

               for( j = 0; j < nCols1; j++ )
               {
                  double value1 = hb_arrayGetND( pRow1, j + 1 );
                  double value2 = hb_arrayGetND( pRow2, j + 1 );
                  hb_arraySetND( pRowResult, j + 1, value1 + value2 );
               }

               hb_arraySet( pMatrixResult, i + 1, pRowResult );
               hb_itemRelease( pRowResult );
            }

            hb_itemReturnRelease( pMatrixResult );
         }
         else
         {
            hb_errRT_BASE( EG_ARG, 3012, "Column dimensions do not match", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
         }
      }
      else
      {
         hb_errRT_BASE( EG_ARG, 3012, "Row dimensions do not match", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
      }
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, "Invalid parameters", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

HB_FUNC( HB_MATRIXSUM )
{
   PHB_ITEM pMatrix = hb_param( 1, HB_IT_ARRAY );
   if( pMatrix )
   {
      int nRows = hb_arrayLen( pMatrix );
      if( nRows > 0 )
      {
         double sum = 0.0;
         int i, j;
         PHB_ITEM pRow;
         int nCols = hb_arrayLen( hb_arrayGetItemPtr( pMatrix, 1 ) );

         for( i = 0; i < nRows; i++ )
         {
            pRow = hb_arrayGetItemPtr( pMatrix, i + 1 );
            for( j = 0; j < nCols; j++ )
            {
               sum += hb_arrayGetND( pRow, j + 1 );
            }
         }

         hb_retnd( sum );
      }
      else
      {
         hb_errRT_BASE( EG_ARG, 3012, "Empty matrix", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
      }
   }
   else
   {
      hb_errRT_BASE( EG_ARG, 3012, "Invalid parameter", HB_ERR_FUNCNAME, HB_ERR_ARGS_BASEPARAMS );
   }
}

#pragma ENDDUMP