Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
045abd6a3c | ||
|
|
06855937bd | ||
|
|
854ff0f904 | ||
|
|
5a5d7a5c59 | ||
|
|
b8949b2ddc | ||
|
|
15d015e59b | ||
|
|
dcf847e2fd | ||
|
|
a3f76e9b0c | ||
|
|
c157a22668 | ||
|
|
cc3707be7a | ||
|
|
75c6f61235 | ||
|
|
8d7692c6df | ||
|
|
318a20c652 | ||
|
|
006cbeed64 | ||
|
|
92ad47fc99 | ||
|
|
84648591ea | ||
|
|
308f99cf63 | ||
|
|
3bfda066ab | ||
|
|
0230f4fa72 | ||
|
|
9f6f366870 | ||
|
|
44a8154cae | ||
|
|
222cd0b562 | ||
|
|
5127510509 | ||
|
|
3876f0e9c3 | ||
|
|
c8d2eb19db | ||
|
|
2943e40e81 | ||
|
|
128deb248d | ||
|
|
151d9faeab | ||
|
|
c335f57ea2 | ||
|
|
c27fd977a8 | ||
|
|
65d01f91b8 | ||
|
|
0427c00015 | ||
|
|
b83de53447 | ||
|
|
5eba3e2558 | ||
|
|
10aa88e03c | ||
|
|
e0e0f9af0f | ||
|
|
35b828a4c5 | ||
|
|
6ebb3b3eae | ||
|
|
569d0e33a6 | ||
|
|
1d1ce5a848 | ||
|
|
f04369343d | ||
|
|
8290abd675 | ||
|
|
ea60d7e172 | ||
|
|
606d59f483 | ||
|
|
a046f286e6 | ||
|
|
0a92a55efa | ||
|
|
3c383559f4 | ||
|
|
6253c7558a | ||
|
|
6835607f66 | ||
|
|
478749566a | ||
|
|
d00c261ff1 | ||
|
|
646a4977f5 | ||
|
|
8683e1d724 | ||
|
|
5a7f1b9bcf | ||
|
|
14089ff59e | ||
|
|
0b0639ce61 | ||
|
|
603ba976e1 | ||
|
|
ad1cea23ab | ||
|
|
b5b5a0e820 | ||
|
|
60666bf79d | ||
|
|
099ffe7e8f | ||
|
|
bd5507d6ef | ||
|
|
3ba8b7b667 | ||
|
|
52dcbf3607 | ||
|
|
e28a537f79 | ||
|
|
b87ac72d78 | ||
|
|
ca367b6918 | ||
|
|
9037f2dd69 | ||
|
|
3a27fde176 | ||
|
|
d3acae2bd8 | ||
|
|
ad0a715a4c | ||
|
|
f7095f8057 | ||
|
|
67fbda60d3 | ||
|
|
5280b84d12 | ||
|
|
b42a81018c | ||
|
|
5c85763de1 | ||
|
|
2aacb60102 | ||
|
|
c4247b8bbc | ||
|
|
c29bafeccc | ||
|
|
fe4bed91d7 | ||
|
|
3abf908ca4 | ||
|
|
9c8e8f54dc | ||
|
|
1e59781d77 | ||
|
|
ead3393211 | ||
|
|
48beb692dc | ||
|
|
fe4b189697 | ||
|
|
c1e79d07e4 | ||
|
|
5de962b2c6 | ||
|
|
bc150bf225 | ||
|
|
31b8a86861 | ||
|
|
dda60313f0 | ||
|
|
fdf33c0f54 | ||
|
|
92bfd8f0ef | ||
|
|
64973a578b | ||
|
|
fb5414efbb | ||
|
|
174cd1c3e7 | ||
|
|
ee6748c261 | ||
|
|
dd0b12213f | ||
|
|
6195f252d9 | ||
|
|
397a60663c | ||
|
|
d792714f4f | ||
|
|
65edbc9e6c | ||
|
|
95df928035 | ||
|
|
78ac84e1cc | ||
|
|
550bcb5cc8 | ||
|
|
44baa0a273 | ||
|
|
78021d6549 | ||
|
|
72d8ce39d6 | ||
|
|
14dd77094d | ||
|
|
031879889c | ||
|
|
a5d2e0d085 | ||
|
|
8403b4fa08 | ||
|
|
757e32229d | ||
|
|
4c995f4c65 | ||
|
|
c950940e22 | ||
|
|
858dfd22e1 | ||
|
|
ab4a4dbd9d | ||
|
|
0bc1aca9bc | ||
|
|
515727ab9b | ||
|
|
3f375bca2a | ||
|
|
483af904af | ||
|
|
bfe0c75106 | ||
|
|
2a96ad3b1d | ||
|
|
12a29b29f6 | ||
|
|
8feab0188c | ||
|
|
fc44efabbb | ||
|
|
3ed09cbc0f | ||
|
|
a4f27bdbe7 | ||
|
|
05d1903045 | ||
|
|
b5b4a00b75 | ||
|
|
97f997840c | ||
|
|
474851b3bb | ||
|
|
4dfa8fd74c | ||
|
|
e709310e70 | ||
|
|
4d0c8ad934 | ||
|
|
b609e8913c | ||
|
|
f10e3d12e6 | ||
|
|
78c514c78b | ||
|
|
d20ae93b40 | ||
|
|
a885153a68 | ||
|
|
3edeb88c92 | ||
|
|
677b174f09 | ||
|
|
ab3d0798c7 | ||
|
|
332983c4d6 | ||
|
|
2c2f23dc57 | ||
|
|
097accc44c | ||
|
|
7ea5614093 | ||
|
|
e4da00cf17 | ||
|
|
03e8abf8c8 | ||
|
|
6750cdf81a | ||
|
|
ffc70e10b9 | ||
|
|
ed834a44b5 | ||
|
|
66312bba2d | ||
|
|
46572501cf | ||
|
|
b9a570424b | ||
|
|
c6c06a7db9 | ||
|
|
f12f69124f | ||
|
|
504a349224 | ||
|
|
d6549d1b86 | ||
|
|
9c9828e9bd | ||
|
|
5b9fe5944b | ||
|
|
3a7b119b24 | ||
|
|
5ad6b68fef | ||
|
|
37e3ffd586 | ||
|
|
36dacc55da | ||
|
|
3180ef63ce | ||
|
|
bdba956bd2 | ||
|
|
732af9de55 | ||
|
|
564f2a7b11 | ||
|
|
39df53e53a | ||
|
|
c2f3feec7e | ||
|
|
3e042eed2d | ||
|
|
183fd316ef | ||
|
|
58e1c23592 | ||
|
|
3e7e53d2fc | ||
|
|
53b6f27fd2 | ||
|
|
8f5e1f1b0c | ||
|
|
6eb679f234 | ||
|
|
1604245f94 | ||
|
|
5154afed01 | ||
|
|
8c40aba850 | ||
|
|
b9a7824b28 | ||
|
|
883ededa8f | ||
|
|
b9db438034 | ||
|
|
0273990196 | ||
|
|
782d2bea72 | ||
|
|
f08883cb69 | ||
|
|
3150aeecaa | ||
|
|
c9c9a0dd6f | ||
|
|
2283b72b2f | ||
|
|
940a3b4534 | ||
|
|
35a9c7ac80 | ||
|
|
2e8240347f | ||
|
|
d69ff4a6f9 | ||
|
|
4ea4934b3c | ||
|
|
add9252b43 | ||
|
|
45d5745b54 | ||
|
|
c92054fb5c | ||
|
|
2e8d94d0bb | ||
|
|
bfbca9dce4 | ||
|
|
2c69eb71d7 | ||
|
|
c50e0e5d0e | ||
|
|
f2dcf8970e | ||
|
|
e882612249 | ||
|
|
9c9a007419 | ||
|
|
56b70481e8 | ||
|
|
55351bc997 | ||
|
|
33e4d14b81 | ||
|
|
7901c7919d | ||
|
|
3b8402374a | ||
|
|
6635defb2e | ||
|
|
7fb073d462 | ||
|
|
2e59f8786b | ||
|
|
ba6069962a | ||
|
|
375de59e28 | ||
|
|
3b84a86b8f | ||
|
|
a5dac2349d | ||
|
|
da6de52198 | ||
|
|
87016e5581 | ||
|
|
e0e45ac048 | ||
|
|
1b3ba04b5b | ||
|
|
5c5180336b | ||
|
|
73bf863e40 | ||
|
|
06d0e354fe | ||
|
|
a01fba6602 | ||
|
|
54ebf2365c | ||
|
|
24eaa7f9df | ||
|
|
8e3eb3a9d7 | ||
|
|
01ce8b8ec3 | ||
|
|
5dd6809ef0 | ||
|
|
7b8bc52650 | ||
|
|
1758c3b6d2 | ||
|
|
6669633902 | ||
|
|
a65d282c0b | ||
|
|
524b43cc40 | ||
|
|
4e0b27c4d7 | ||
|
|
c6d7acfd99 | ||
|
|
bf941c71ec | ||
|
|
dcec510451 | ||
|
|
cb468b0f3c | ||
|
|
6713510ce9 | ||
|
|
bf5b32de16 | ||
|
|
2a1780b43d | ||
|
|
be05ec99b5 | ||
|
|
3a42bc034d | ||
|
|
27e7664ef0 | ||
|
|
ccbf1d3269 | ||
|
|
0ca14739f9 | ||
|
|
d9192ef355 | ||
|
|
4c9bfd576c | ||
|
|
20898cade7 | ||
|
|
2c12639499 | ||
|
|
7d2b2067ef | ||
|
|
0b52e6ee86 | ||
|
|
cafac69008 | ||
|
|
871875b8e5 | ||
|
|
e5ae3e48d6 | ||
|
|
76de3dd75e | ||
|
|
687427381d | ||
|
|
e418d87802 | ||
|
|
a1f3690085 | ||
|
|
0a7bed10bf | ||
|
|
32996ce9aa | ||
|
|
b42265b41e | ||
|
|
0fc7654f6c | ||
|
|
effad97472 | ||
|
|
53d98bfffe | ||
|
|
62aa6079be | ||
|
|
7d48f65ff9 | ||
|
|
3acadf6ad7 | ||
|
|
5492f99f06 | ||
|
|
71221e055b | ||
|
|
86330e8f65 | ||
|
|
f3b507b8d9 | ||
|
|
c01c7e862d | ||
|
|
99cd765997 | ||
|
|
5266506b56 | ||
|
|
682a5030d3 | ||
|
|
fc3d847d85 | ||
|
|
82756030c6 | ||
|
|
0352451ead | ||
|
|
593cf492c3 | ||
|
|
0b8ad8b0f3 | ||
|
|
111d927ece | ||
|
|
2dae023ee3 | ||
|
|
b12122f78a | ||
|
|
0c7b3162c8 | ||
|
|
d4d050297b | ||
|
|
8d768bf9d5 | ||
|
|
47ac8bf6c0 | ||
|
|
01572736c0 | ||
|
|
b6e5f01748 | ||
|
|
90da0ac990 | ||
|
|
bc26d1506b | ||
|
|
769e3ca559 | ||
|
|
b5d789f02a | ||
|
|
c8dce25e0d | ||
|
|
b322e2d24a | ||
|
|
414ec69c0a | ||
|
|
f5ca656c04 | ||
|
|
e158232aba | ||
|
|
632871ac24 | ||
|
|
7af7b2c8c2 | ||
|
|
97132770e4 | ||
|
|
683ad3ef94 | ||
|
|
604383f51c | ||
|
|
bdeb2f00b3 | ||
|
|
8688875fae | ||
|
|
a42271c7de | ||
|
|
cf4685eaf1 | ||
|
|
5023fb6774 | ||
|
|
04eb70cbe4 | ||
|
|
4dc010f54d | ||
|
|
f7e36bc44e | ||
|
|
a9ac1c06a7 | ||
|
|
9bf6e7492f | ||
|
|
14070cef43 | ||
|
|
6b1abbb50e | ||
|
|
46c641ebc5 | ||
|
|
2d036f5126 | ||
|
|
2c69e357b8 | ||
|
|
ba81f22d40 | ||
|
|
070add4e7b | ||
|
|
847b3225b2 | ||
|
|
3ae5c7efac | ||
|
|
4966c27548 | ||
|
|
06c12d0383 | ||
|
|
69f3968d4a | ||
|
|
d5ae66466a | ||
|
|
2dc8be3127 | ||
|
|
040e80a36d | ||
|
|
bd6b79fb41 | ||
|
|
581a31161c | ||
|
|
5f8f93ba6c | ||
|
|
40b3e9c5dd | ||
|
|
2ffc219f47 | ||
|
|
02394d3278 | ||
|
|
ff1ccca3cd | ||
|
|
fddbaec82e | ||
|
|
8e073438a3 | ||
|
|
6972e9ec87 | ||
|
|
f29a22886a | ||
|
|
287afa1537 | ||
|
|
b0ac6dfdb9 | ||
|
|
7f06606300 | ||
|
|
985af3a294 | ||
|
|
440cfdcc31 | ||
|
|
0ca92c3723 | ||
|
|
f434f30286 | ||
|
|
f4b8935189 | ||
|
|
014df322e7 | ||
|
|
39f9cc16e4 | ||
|
|
b4b3032456 | ||
|
|
edef1ea708 | ||
|
|
7523670949 | ||
|
|
d81f328859 | ||
|
|
1a39d601de | ||
|
|
68d3dd85a9 | ||
|
|
cb80b08f77 | ||
|
|
e5577b9996 | ||
|
|
9bc31d069b | ||
|
|
411667fd88 | ||
|
|
ed9029d760 | ||
|
|
1892062d88 | ||
|
|
cbe3d9bc75 | ||
|
|
2ebfe74459 | ||
|
|
6257adb46b | ||
|
|
31a0ab5b70 | ||
|
|
f783394f09 | ||
|
|
2be0b15bcf | ||
|
|
de6bde1613 | ||
|
|
fdc9f47a0d | ||
|
|
49c715199d | ||
|
|
fd192df036 | ||
|
|
8662d23ec8 | ||
|
|
10453d8dfe | ||
|
|
8235d632b8 | ||
|
|
f4c8504c8d | ||
|
|
4fb7c43e06 | ||
|
|
54caa48d1f | ||
|
|
f00a8bea8f | ||
|
|
55a9fcc0f4 | ||
|
|
4cdea956d7 | ||
|
|
ae06436f91 | ||
|
|
d0e10d4cc6 | ||
|
|
c94f065f80 | ||
|
|
36b4962efb | ||
|
|
e01e837a2a | ||
|
|
634ba8a272 | ||
|
|
ce1944d437 | ||
|
|
39bad990c0 | ||
|
|
431d6e9338 | ||
|
|
d453969708 | ||
|
|
778fa65b4c | ||
|
|
69c3817579 | ||
|
|
b89af917b1 | ||
|
|
d79202e698 | ||
|
|
4ce677902a | ||
|
|
fe9f7b49f7 | ||
|
|
649f965c52 | ||
|
|
d6fa8c7bb1 | ||
|
|
d2734dc8ce | ||
|
|
bab272bb5c | ||
|
|
34428ea7b3 | ||
|
|
83e6a951dd | ||
|
|
3d618eb660 | ||
|
|
2add24a9c1 | ||
|
|
a50227849d | ||
|
|
5ffc9d851a | ||
|
|
1e0e419210 | ||
|
|
2813d546a7 | ||
|
|
19e94dfa51 | ||
|
|
49f97fc26e | ||
|
|
562290d922 | ||
|
|
66a5505965 | ||
|
|
c5c225a68e | ||
|
|
1bacfd69d9 | ||
|
|
383ffa6a65 | ||
|
|
69927176df | ||
|
|
5c2ae26a74 | ||
|
|
3f52b56251 | ||
|
|
72da123a51 | ||
|
|
e94af0369b | ||
|
|
8da65611cb | ||
|
|
605060c51d | ||
|
|
856e654306 | ||
|
|
4de9ee038c | ||
|
|
290d62da4e | ||
|
|
140e9a679d | ||
|
|
267a33d43b | ||
|
|
435a0687d7 | ||
|
|
dbf935bec0 | ||
|
|
d6d081c0e9 | ||
|
|
c72f852a55 | ||
|
|
99779ecc08 | ||
|
|
c4b8e66206 | ||
|
|
ca6407e0c0 | ||
|
|
dff32a97ee | ||
|
|
d5c528fead | ||
|
|
38f39a10ae | ||
|
|
48e023ba25 | ||
|
|
011ec84f23 | ||
|
|
472ea9d309 | ||
|
|
6ba04c4381 | ||
|
|
06f8e63d06 | ||
|
|
31d4616baf | ||
|
|
4c4b1222d5 | ||
|
|
a3973b981c | ||
|
|
b6e2029770 | ||
|
|
6b878caff3 | ||
|
|
e9b7a317c1 | ||
|
|
14b8032635 | ||
|
|
7dfe26c83d | ||
|
|
24229316c7 | ||
|
|
7d70e0fb75 | ||
|
|
d70ecd0968 | ||
|
|
36a4030c09 | ||
|
|
330462e7a5 | ||
|
|
922dfa2e79 | ||
|
|
34fce932cb | ||
|
|
18b1e13f34 | ||
|
|
7bee4a237d | ||
|
|
d6cca2b1af | ||
|
|
58312e64da | ||
|
|
3b0927a8c9 | ||
|
|
6e527c371d | ||
|
|
6cccdc1f34 | ||
|
|
0abc17e9f2 | ||
|
|
b83d8dacdf | ||
|
|
dadc055429 | ||
|
|
750114a923 | ||
|
|
83a81da319 | ||
|
|
3eb0256127 | ||
|
|
9d8b3cc854 | ||
|
|
0ab402aa73 | ||
|
|
c36c299466 | ||
|
|
8611416670 | ||
|
|
e20fd6bf97 | ||
|
|
943005328c | ||
|
|
e1842043d8 | ||
|
|
9fb094f505 | ||
|
|
d142049216 | ||
|
|
33fe1a3439 | ||
|
|
a726ad8511 | ||
|
|
b926f01b85 | ||
|
|
8264e7ffae | ||
|
|
80852374a8 | ||
|
|
e3c6789776 | ||
|
|
e2417cb5ab | ||
|
|
eccbf7474e | ||
|
|
69cee4d93a | ||
|
|
705f755b6b | ||
|
|
8ad8eaac71 | ||
|
|
dd9224e685 | ||
|
|
aeb7516445 | ||
|
|
bca0fdd941 | ||
|
|
60da957f13 | ||
|
|
f621620593 | ||
|
|
eff74cabe0 | ||
|
|
6f0a6f7fd8 | ||
|
|
ab7c227b97 | ||
|
|
b8054665bc | ||
|
|
99a5b735c8 | ||
|
|
fb2d83e9e3 | ||
|
|
82f6b4a791 | ||
|
|
7e1f5d6852 | ||
|
|
ab766862a3 | ||
|
|
2bd9dd7535 | ||
|
|
7f0f64a42e | ||
|
|
f7e068baed | ||
|
|
2e2a33cef3 | ||
|
|
2c460022f8 | ||
|
|
133644a0ba | ||
|
|
ba0ec3d1fa | ||
|
|
22e9240e4f | ||
|
|
6a58a59a11 | ||
|
|
26328fadeb | ||
|
|
c0eea526de | ||
|
|
94ea5909f4 | ||
|
|
3758db2861 | ||
|
|
8b09093aca | ||
|
|
61347e0f0b | ||
|
|
3131277b19 | ||
|
|
23a3c147cd | ||
|
|
f02174af57 | ||
|
|
e3434d19ea | ||
|
|
62cff271d8 | ||
|
|
56b46cde0e | ||
|
|
75b8d294b0 | ||
|
|
634d10cdd4 | ||
|
|
0d4f43a8bf | ||
|
|
4231f2e929 | ||
|
|
8f26a418a9 | ||
|
|
f50a9f5bf3 | ||
|
|
8e2b6203a1 | ||
|
|
c9e3240ae2 | ||
|
|
0841778b99 | ||
|
|
788d84a2bf | ||
|
|
168261964e | ||
|
|
2f129c3f23 | ||
|
|
d98c0c2f33 | ||
|
|
3261917d62 | ||
|
|
e3df4bbf00 | ||
|
|
f630771150 | ||
|
|
431fb5626d | ||
|
|
19e12fa22e | ||
|
|
daf9ff3b2d | ||
|
|
1ed0d52619 | ||
|
|
0e9fcb78e0 | ||
|
|
19d5b931c4 | ||
|
|
18b7dee2f6 | ||
|
|
731b3e4b47 | ||
|
|
cdb4d29676 | ||
|
|
256f5a4a03 | ||
|
|
b69cb9b0f8 | ||
|
|
140a8f6efe | ||
|
|
14b99826eb | ||
|
|
0201d963b7 | ||
|
|
9be0fe6626 | ||
|
|
24b5dd033a | ||
|
|
686a6da019 | ||
|
|
bb0c8e7731 | ||
|
|
d3f7a04411 | ||
|
|
96bb1cbcae | ||
|
|
93fc562c44 | ||
|
|
c59f6b5fca | ||
|
|
ba7b49c35b | ||
|
|
1db3e0ad2f | ||
|
|
150b57d538 | ||
|
|
ce2f0b6a6c | ||
|
|
c3e6841293 | ||
|
|
12a53d647e | ||
|
|
314e37c433 | ||
|
|
05ed36240b | ||
|
|
520059d866 | ||
|
|
162a5b4acc | ||
|
|
d47fcbf2be | ||
|
|
cb07b92543 | ||
|
|
9274dbd49f | ||
|
|
d6cdd670b1 | ||
|
|
a5201a62e0 | ||
|
|
ee4c273d73 | ||
|
|
eea2108ac2 | ||
|
|
7f0e34d318 | ||
|
|
d6b7c0e9cb | ||
|
|
f049e208b6 | ||
|
|
88bb817a62 | ||
|
|
8ddee212e5 | ||
|
|
d3299166bb | ||
|
|
31d4be8015 | ||
|
|
45be3125de | ||
|
|
063b02e996 | ||
|
|
3f7b7847a5 | ||
|
|
7d87ad387b | ||
|
|
9dce341bc8 | ||
|
|
88eecd7671 | ||
|
|
83c412b33d | ||
|
|
31b65ade5b | ||
|
|
7cf7d33b6d | ||
|
|
4c4e415975 | ||
|
|
c55e3e0cbc | ||
|
|
240fd8586e | ||
|
|
ce23ab38f7 | ||
|
|
55696bfb31 | ||
|
|
400224a089 | ||
|
|
5c1823d6d2 | ||
|
|
a3642caa3d | ||
|
|
e2153435dc | ||
|
|
f307ecb380 | ||
|
|
50b823e76d | ||
|
|
84bda90065 | ||
|
|
d6fdcef4d4 | ||
|
|
62023acd4d | ||
|
|
75ef597ece | ||
|
|
0cbbbf260a | ||
|
|
4f99d5ce61 | ||
|
|
f96c4ccd64 | ||
|
|
ffa07b7df4 | ||
|
|
e81dd9a7ec | ||
|
|
30f2df3004 | ||
|
|
ac16fc1f0e | ||
|
|
e4aca3b31e | ||
|
|
212753f311 | ||
|
|
ccfcb58ed1 | ||
|
|
aa3df74fc1 | ||
|
|
f9d2c0d2d4 | ||
|
|
c066b2c82a | ||
|
|
ab795ec9e0 | ||
|
|
7042307955 | ||
|
|
f37d5fda3c | ||
|
|
c5b92eabe1 | ||
|
|
01453bce5f | ||
|
|
bd86ba36e2 | ||
|
|
88ab8db88d | ||
|
|
11406034c9 | ||
|
|
eef8671912 | ||
|
|
72383b1634 | ||
|
|
2d70be0ae2 | ||
|
|
5350738c22 | ||
|
|
0d29dd00fd | ||
|
|
d441481930 | ||
|
|
7ff9b854b6 | ||
|
|
46be24f6a3 | ||
|
|
3908e1601a | ||
|
|
1ef0a7ed82 | ||
|
|
43f5b4a078 | ||
|
|
83355a25c8 | ||
|
|
ec56bc32f8 | ||
|
|
050c3d2515 | ||
|
|
b649c6b301 | ||
|
|
99c9acccb0 | ||
|
|
524e6da591 | ||
|
|
6292bfd9cb | ||
|
|
83063d3a18 | ||
|
|
7b0dacabdd | ||
|
|
c8b861f0b6 | ||
|
|
09912509cb | ||
|
|
3e0940da8c | ||
|
|
d20e623f0c | ||
|
|
ef7a3ff1ed | ||
|
|
2115cb9085 | ||
|
|
5f981509ac |
@@ -15,6 +15,7 @@ docker-compose.yml
|
||||
.env.*
|
||||
*.log
|
||||
.obsigate-backup
|
||||
backend/VERSION
|
||||
.pytest_cache
|
||||
htmlcov
|
||||
.coverage
|
||||
|
||||
+110
-5
@@ -7,16 +7,36 @@ OBSIGATE_AUTH_ENABLED=true
|
||||
OBSIGATE_ADMIN_USER=admin
|
||||
OBSIGATE_ADMIN_PASSWORD=chab30
|
||||
|
||||
# Sécurité des cookies (activer si derrière HTTPS)
|
||||
# OBSIGATE_SECURE_COOKIES=false
|
||||
# DANGER : si OBSIGATE_AUTH_ENABLED=false, toute requête devient un admin
|
||||
# anonyme. Le serveur REFUSE de démarrer sur une adresse non-loopback
|
||||
# (ex. 0.0.0.0) sauf si l'on force l'opt-in ci-dessous. À réserver au local.
|
||||
# OBSIGATE_ALLOW_INSECURE=false
|
||||
|
||||
# Sécurité des cookies : true|false|auto (défaut : auto — Secure si la
|
||||
# requête arrive en https, sinon pas de flag ; les navigateurs ignorent les
|
||||
# cookies `Secure` en HTTP, ce qui casserait les logins en local).
|
||||
# Derrière un reverse proxy qui termine TLS, auto suffit avec
|
||||
# OBSIGATE_TRUST_PROXY=true (X-Forwarded-Proto honoré).
|
||||
# OBSIGATE_SECURE_COOKIES=auto
|
||||
|
||||
# Tokens TTL en secondes
|
||||
# OBSIGATE_ACCESS_TOKEN_TTL=900
|
||||
# OBSIGATE_ACCESS_TOKEN_TTL=31536000000 # 1000 ans
|
||||
# OBSIGATE_REFRESH_TOKEN_TTL=604800
|
||||
|
||||
# Rate limiting
|
||||
# OBSIGATE_LOGIN_MAX_ATTEMPTS=10
|
||||
# OBSIGATE_ACCOUNT_MAX_ATTEMPTS=10
|
||||
# OBSIGATE_LOGIN_WINDOW_SECONDS=900
|
||||
# Compteurs partagés/persistants (SQLite WAL, multi-workers) — défaut : mémoire.
|
||||
# OBSIGATE_RATELIMIT_DB=data/ratelimit.db
|
||||
|
||||
# IP client derrière un reverse proxy (fait confiance à X-Forwarded-For)
|
||||
# OBSIGATE_TRUST_PROXY=false
|
||||
|
||||
# Webhooks : sécurité SSRF
|
||||
# OBSIGATE_WEBHOOK_ALLOW_HTTP=false # autoriser http:// (défaut : HTTPS requis)
|
||||
# OBSIGATE_WEBHOOK_ALLOW_PRIVATE=false # autoriser les IP privées/boucle
|
||||
# Secret d'un webhook : OBSIGATE_WEBHOOK_SECRET_<ID_WEBHOOK_EN_MAJUSCULES>
|
||||
|
||||
# Watcher
|
||||
# OBSIGATE_WATCHER_ENABLED=true
|
||||
@@ -27,20 +47,42 @@ OBSIGATE_ADMIN_PASSWORD=chab30
|
||||
# Ignored directories (séparés par des virgules)
|
||||
# OBSIGATE_IGNORED_DIRS=.obsidian,.trash,.git,__pycache__,node_modules,.obsigate-backup
|
||||
|
||||
# Vaults DIR (DIR_N_NAME / DIR_N_PATH dans le compose) — options par vault :
|
||||
# DIR_N_READONLY=true : lecture seule déclarée (mutations refusées
|
||||
# en 403, actions d'écriture masquées dans l'UI)
|
||||
# DIR_N_IGNORED_DIRS=.home,.venv : noms de dossiers cachés pour CE vault
|
||||
# (toute profondeur, en plus du global)
|
||||
# DIR_N_IGNORED_PATHS=Executeurs/*/data,Executeurs/*/.env
|
||||
# : motifs fnmatch sur le chemin relatif dans
|
||||
# le vault (un dossier qui matche cache son
|
||||
# sous-arbre) — navigation, lecture, index
|
||||
|
||||
# Audit
|
||||
# OBSIGATE_AUDIT_MAX_SIZE=10485760
|
||||
|
||||
# Backup
|
||||
# OBSIGATE_BACKUP_DIR=.obsigate-backup
|
||||
|
||||
# PDF (ROADMAP #74)
|
||||
# OBSIGATE_PDF_MAX_SIZE_MB=50 # PDFs plus volumineux = texte non indexé
|
||||
# OBSIGATE_PDF_EXTRACT_TIMEOUT=30 # secondes avant abandon de l'extraction
|
||||
|
||||
# WebAuthn / MFA (ROADMAP #64) — par défaut rp_id/origines sont dérivés de la
|
||||
# requête (hôte exact, port inclus) : rien à configurer en accès direct.
|
||||
# À renseigner uniquement pour un accès via reverse-proxy sous un autre nom
|
||||
# (avec OBSIGATE_TRUST_PROXY=true pour X-Forwarded-Host/Proto) :
|
||||
# OBSIGATE_WEBAUTHN_RP_ID=obsigate.example.com
|
||||
# OBSIGATE_WEBAUTHN_RP_NAME=ObsiGate
|
||||
# OBSIGATE_WEBAUTHN_ORIGINS=https://obsigate.example.com
|
||||
|
||||
# ── AI Provider Configuration ──
|
||||
# Définir au moins un provider pour activer les fonctionnalités AI dans l'éditeur
|
||||
|
||||
# AI_DEFAULT_PROVIDER=deepseek # deepseek | openrouter | gemini
|
||||
|
||||
# DeepSeek (recommandé, bon marché)
|
||||
DEEPSEEK_API_KEY=sk-87d93019602e4c279679dfe80d504bc3
|
||||
DEEPSEEK_MODEL=deepseek-v4-pro
|
||||
DEEPSEEK_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
DEEPSEEK_MODEL=deepseek-chat
|
||||
|
||||
# OpenRouter (accès à plusieurs modèles)
|
||||
# OPENROUTER_API_KEY=sk-or-v1-...
|
||||
@@ -49,3 +91,66 @@ DEEPSEEK_MODEL=deepseek-v4-pro
|
||||
# Google Gemini
|
||||
# GEMINI_API_KEY=AIza...
|
||||
# GEMINI_MODEL=gemini-2.0-flash
|
||||
|
||||
# ── Assistant IA — recherche web (outil web_search) ──
|
||||
# Instance SearXNG auto-hébergée (aucune clé API requise)
|
||||
# OBSIGATE_SEARXNG_URL=https://search.dracodev.net
|
||||
# Chaîne de repli sans clé (DuckDuckGo puis Bing) si SearXNG ne remonte rien
|
||||
# OBSIGATE_WEB_FALLBACK=1
|
||||
# OBSIGATE_WEB_TIMEOUT=10
|
||||
# Fournisseurs à clé (#92), essayés avant SearXNG — injecter via Infisical en prod
|
||||
# OBSIGATE_TAVILY_API_KEY=
|
||||
# OBSIGATE_BRAVE_API_KEY=
|
||||
# OBSIGATE_SERPAPI_API_KEY=
|
||||
# OBSIGATE_EXA_API_KEY=
|
||||
# Ordre des fournisseurs (sinon : clés présentes puis SearXNG puis replis)
|
||||
# OBSIGATE_WEB_PROVIDERS=brave,searxng
|
||||
# Réessais réseau (backoff maison) + cache SQLite des résultats web
|
||||
# OBSIGATE_WEB_RETRY=1
|
||||
# OBSIGATE_WEB_CACHE_TTL=900 # secondes ; 0 = cache désactivé
|
||||
# Rendu dynamique (pages SPA) — dépendance optionnelle :
|
||||
# pip install playwright && playwright install chromium
|
||||
# ── Assistant IA — sources connectées (Gitea / GitHub) ──
|
||||
# OBSIGATE_GITEA_URL=https://git.example.net
|
||||
# OBSIGATE_GITEA_TOKEN=
|
||||
# OBSIGATE_GITHUB_TOKEN=
|
||||
|
||||
# ── Fusion / dev-prod — URL publiques de l'écosystème (#223) ──
|
||||
# Aucun nom public n'est figé dans le code : ces variables portent les
|
||||
# URL des autres applications. Valeurs par défaut (si absentes) = noms
|
||||
# OFFICIELS de la production. Depuis BUG-120, les URL sœurs se dérivent
|
||||
# de l'hôte de l'agent (agent<SUFFIXE>.dracodev.net, lu via
|
||||
# AGENT_EVENEMENTS_URL / DRACO_AGENT_URL) : en dev, rien d'autre à
|
||||
# déclarer pour obtenir les noms en -dev. Une variable explicite
|
||||
# reste prioritaire sur la dérivation, par exemple :
|
||||
# DRACO_TIKTAK_URL=https://tiktak-dev.dracodev.net
|
||||
# DRACO_AGENT_URL=https://agent.dracodev.net
|
||||
# DRACO_OBSIGATE_URL=https://og.dracodev.net
|
||||
# DRACO_FLOWDECK_URL=https://flowdeck.dracodev.net
|
||||
# DRACO_TIKTAK_URL=https://tiktak.dracodev.net
|
||||
# DRACO_KANBANTE_URL=https://kanbante.dracodev.net
|
||||
# DRACO_SHAARIT_URL=https://shaarit.dracodev.net
|
||||
# DRACO_HUB_URL=https://hub.dracodev.net
|
||||
# DRACO_GITEA_URL=https://git.dracodev.net
|
||||
|
||||
# ── Fusion — émission d'événements vers le service d'agents (#220) ──
|
||||
# URL complète de réception ; à défaut, dérivée de DRACO_AGENT_URL.
|
||||
# AGENT_EVENEMENTS_URL=https://agent.dracodev.net/api/v1/evenements
|
||||
# AGENT_EVENEMENTS_CLE=<clé propre à ObsiGate, jamais commitée>
|
||||
# AGENT_EVENEMENTS_DOMAINES={"NomDuVault": "famille"}
|
||||
|
||||
# ── Fusion — émettrice SSO draco_session (#216/#217) ──
|
||||
# DRACO_SESSION_SECRET=<secret partagé, jamais commité>
|
||||
# DRACO_SESSION_SECRET_PREV=<ancien secret, fenêtre de rotation>
|
||||
# DRACO_SESSION_COOKIE_DOMAIN=.dracodev.net
|
||||
# DRACO_SSO_MACHINE_KEY=<clé machine des révocations, jamais commitée>
|
||||
|
||||
# ── Coffre Secrets ObsiGate (Phases 1–2, voir docs/COFFRE-sauvegarde-restauration.md) ──
|
||||
# Activation (défaut : désactivé — toutes les routes répondent 503 sinon) :
|
||||
# OBSIGATE_SECRETS_ENABLED=true
|
||||
# Clé maître 256 bits, base64 de 32 octets, rendue une seule fois par
|
||||
# POST /api/coffres/admin/generer-kek ; à sauvegarder HORS dépôt, HORS note :
|
||||
# OBSIGATE_KEK=<kek-base64-32-octets>
|
||||
# … ou fichier 0600, hors dépôt et hors arborescence des vaults :
|
||||
# OBSIGATE_KEK_FILE=/chemin/hors/depot/kek.txt
|
||||
# Aucune autre variable ne doit contenir une valeur de secret du coffre.
|
||||
|
||||
+194
-30
@@ -12,7 +12,7 @@ on:
|
||||
jobs:
|
||||
# ── Lint ──────────────────────────────────────────────────────────
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ci-deepin
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -30,18 +30,77 @@ jobs:
|
||||
run: ruff check backend/
|
||||
|
||||
- name: Mypy (type checker)
|
||||
run: mypy backend/ --ignore-missing-imports || echo "mypy found type errors (advisory — 28 pre-existing issues)"
|
||||
run: mypy backend/ --ignore-missing-imports
|
||||
|
||||
- name: Frontend validation
|
||||
run: node tests/frontend/validate-imports.mjs
|
||||
|
||||
- name: Frontend unit tests
|
||||
run: node tests/frontend/unit.test.mjs
|
||||
run: |
|
||||
node tests/frontend/unit.test.mjs
|
||||
node tests/frontend/navfacets.test.mjs
|
||||
node tests/frontend/desktop-roots.test.mjs
|
||||
node tests/frontend/image-viewer.test.mjs
|
||||
node tests/frontend/pdf-viewer.test.mjs
|
||||
node tests/frontend/forge-completion.test.mjs
|
||||
node tests/frontend/config-mobile.test.mjs
|
||||
node tests/frontend/settings-order-avatar.test.mjs
|
||||
node tests/frontend/mobile-toolbar.test.mjs
|
||||
node tests/frontend/pretty.test.mjs
|
||||
node tests/frontend/media-viewer.test.mjs
|
||||
node tests/frontend/mfa-settings.test.mjs
|
||||
|
||||
- name: Frontend JSDOM tests (PaneManager + Excalidraw + Plugins + AI + SW + Collab + Mobile + Semantic + Desktop + Inline edition + Upload + XLSX)
|
||||
run: |
|
||||
cd tests/frontend
|
||||
if [ -d node_modules ]; then
|
||||
node pane-manager.test.mjs
|
||||
node excalidraw-viewer.test.mjs
|
||||
node plugins.test.mjs
|
||||
node ai.test.mjs
|
||||
node ai-sidebar.test.mjs
|
||||
node sidebar-filters.test.mjs
|
||||
node search-facets.test.mjs
|
||||
node sw.test.mjs
|
||||
node collab.test.mjs
|
||||
node mobile-editor.test.mjs
|
||||
node semantic-search.test.mjs
|
||||
node desktop.test.mjs
|
||||
node toolbar-order.test.mjs
|
||||
node editor-inline.test.mjs
|
||||
node ai-quick-actions.test.mjs
|
||||
node upload.test.mjs
|
||||
node config-ai-keys.test.mjs
|
||||
node xlsx-viewer.test.mjs
|
||||
node filechat.test.mjs
|
||||
else
|
||||
echo "tests/frontend/node_modules missing - installing jsdom"
|
||||
npm install --no-audit --no-fund --silent
|
||||
node pane-manager.test.mjs
|
||||
node excalidraw-viewer.test.mjs
|
||||
node plugins.test.mjs
|
||||
node ai.test.mjs
|
||||
node ai-sidebar.test.mjs
|
||||
node sidebar-filters.test.mjs
|
||||
node search-facets.test.mjs
|
||||
node sw.test.mjs
|
||||
node collab.test.mjs
|
||||
node mobile-editor.test.mjs
|
||||
node semantic-search.test.mjs
|
||||
node desktop.test.mjs
|
||||
node toolbar-order.test.mjs
|
||||
node editor-inline.test.mjs
|
||||
node ai-quick-actions.test.mjs
|
||||
node upload.test.mjs
|
||||
node config-ai-keys.test.mjs
|
||||
node xlsx-viewer.test.mjs
|
||||
node filechat.test.mjs
|
||||
fi
|
||||
|
||||
# ── Tests ─────────────────────────────────────────────────────────
|
||||
test:
|
||||
needs: lint
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ci-deepin
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -54,6 +113,7 @@ jobs:
|
||||
run: |
|
||||
pip install pytest pytest-cov pytest-asyncio httpx
|
||||
pip install -r backend/requirements.txt
|
||||
pip install -r backend/requirements-test.txt || echo "test deps install failed (non-blocking — PDF tests will skip)"
|
||||
|
||||
- name: Run tests
|
||||
run: pytest tests/ --cov=backend --cov-report=xml --cov-report=term -q
|
||||
@@ -67,7 +127,7 @@ jobs:
|
||||
|
||||
# ── Security scan ─────────────────────────────────────────────────
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ci-deepin
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -77,30 +137,114 @@ jobs:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install dependencies
|
||||
# setuptools / pip sont mis à jour : l'image de base peut embarquer
|
||||
# une version couverte par un advisory fraîchement publié
|
||||
# (PYSEC-2026-3447 / PYSEC-2026-3721).
|
||||
# NOTE runner Gitea Act (BUG-083) : aucun `#` dans le `run:`.
|
||||
run: |
|
||||
pip install -U pip setuptools
|
||||
pip install bandit pip-audit
|
||||
pip install -r backend/requirements.txt
|
||||
|
||||
- name: Bandit (SAST)
|
||||
run: bandit -r backend/ --skip B101,B110,B310 || echo "bandit found issues (non-blocking)"
|
||||
- name: Bandit (SAST, bloquant — #87)
|
||||
# B105 est exclu (aligné avec [tool.bandit] de pyproject.toml :
|
||||
# faux positifs systématiques sur les noms de variables) ; les rares
|
||||
# vrais positifs restants portent un `# nosec` justifié inline.
|
||||
run: bandit -r backend/ --skip B101,B105,B110,B310
|
||||
|
||||
- name: Pip-audit (dependency vulnerabilities)
|
||||
run: pip-audit || echo "pip-audit found vulnerabilities (non-blocking)"
|
||||
- name: Semgrep (SAST local) — DÉSACTIVÉ (BUG-091)
|
||||
# Les règles locales (semgrep-rules/, 8 règles) ne sont plus exécutées
|
||||
# en CI : semgrep-core est un exécutable natif que le runner actuel ne
|
||||
# peut pas lancer (exit 127, sans message exploitable) — les releases
|
||||
# récentes exigent un CPU x86-64-v2, et la dernière version compatible
|
||||
# (1.157.0, core statique vérifié en baseline v1) échoue aussi. Les
|
||||
# règles restent applicables en local : `semgrep --config semgrep-rules/
|
||||
# backend/`. À réactiver dès que le runner dispose d'un CPU x86-64-v2
|
||||
# (ou d'une image de runner plus récente). Bandit et pip-audit, eux,
|
||||
# restent bloquants dans ce job.
|
||||
# NOTE runner Gitea Act (BUG-083) : aucun `#` dans le `run:`.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "::warning::SAST semgrep non exécutée (runner incompatible — BUG-091). Bandit et pip-audit restent bloquants."
|
||||
|
||||
- name: Pip-audit (bloquant — #87)
|
||||
# Bloquant depuis T6 (#87) : dépendances qualifiées (mistune 3.3.3,
|
||||
# python-multipart 0.0.31, weasyprint 70, mcp 1.28.1, fastapi 0.141.1
|
||||
# + starlette 1.7.0, setuptools 84 — suite complète verte + 0 vuln).
|
||||
# Seule exception documentée : PYSEC-2026-1325 (ecdsa, Minerva) —
|
||||
# aucun correctif upstream ET ObsiGate ne signe/vérifie qu'en HS256
|
||||
# (backend/auth/jwt_handler.py), les chemins ECDSA P-256 ne
|
||||
# s'exécutent jamais. Les advisories pyjwt (PYSEC-2026-178 puis
|
||||
# CVE-2026-102274) sont corrigées par le plancher pyjwt>=2.14.0 de
|
||||
# backend/requirements.txt (BUG-091, BUG-095).
|
||||
# PYSEC-2026-3910 / PYSEC-2026-3911 (pypdf, DoS de ressources sur
|
||||
# l'extraction de texte et la lecture d'outlines — donc atteignables
|
||||
# via backend/pdf_reader.py) sont corrigés par le plancher
|
||||
# pypdf>=6.16.1 (BUG-093), relevé à pypdf>=6.21.2 par #224 (11 CVE
|
||||
# sur la 6.19.0 de la toolcache, corrigées au plus tard en 6.21.2).
|
||||
# #224 ajoute les planchers brotli>=1.2.1 (PYSEC-2026-4322,
|
||||
# GHSA-9v8h-qqgm-4m35), filelock>=4.2.1 (CVE-2026-112940,
|
||||
# CVE-2026-112913, CVE-2026-112907) et opentelemetry-api>=1.45.1
|
||||
# (CVE-2026-114603) : paquets transitifs ou préinstallés dans la
|
||||
# toolcache, audités mais jamais alignés sans plancher explicite.
|
||||
# CVE-2026-97687 / CVE-2026-97688 / CVE-2026-97689 (urllib3 2.7.0)
|
||||
# corrigés par le plancher urllib3>=2.8.0.
|
||||
# CVE-2026-104874 (multidict 6.7.x) corrigé par le plancher
|
||||
# multidict>=6.9.1 (transitive aiohttp/yarl ; 6.7.x est dans la
|
||||
# toolcache de l'image du runner — même piège « already satisfied »).
|
||||
# CVE-2026-85394 (python-jose ≤3.5.0, forgery HS256 par clé publique
|
||||
# DER passée comme secret HMAC) : AUCUN correctif upstream (projet
|
||||
# sans release depuis 2025). Non atteignable dans ObsiGate :
|
||||
# jwt.decode passe toujours algorithms=["HS256"] et un secret
|
||||
# symétrique serveur (backend/auth/jwt_handler.py,
|
||||
# backend/mcp/confirmations.py) — jamais une clé publique comme clé.
|
||||
# Ces planchers doivent rester *au-dessus* des versions préinstallées
|
||||
# dans la toolcache de l'image du runner : en dessous, pip répond
|
||||
# « already satisfied » et n'aligne jamais (c'est exactement ce qui a
|
||||
# fait échouer ce job). Le garde-fou tests/test_ci_workflow.py::
|
||||
# TestDependencySecurityFloors verrouille ces planchers.
|
||||
# NOTE runner Gitea Act (BUG-083) : aucun `#` dans le `run:`.
|
||||
run: pip-audit --ignore-vuln PYSEC-2026-1325 --ignore-vuln CVE-2026-85394
|
||||
|
||||
# ── Docker build ──────────────────────────────────────────────────
|
||||
build:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ci-deepin
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Generate VERSION file
|
||||
run: |
|
||||
VERSION=$(git describe --tags --dirty 2>/dev/null | sed 's/^v//' || echo "0.0.0-dev")
|
||||
echo "$VERSION" > backend/VERSION
|
||||
- name: Version livrée
|
||||
# VERSION (racine du dépôt) est copié dans l'image par le Dockerfile :
|
||||
# plus aucun numéro généré ni codé en dur dans le pipeline.
|
||||
run: echo "Version livree = $(cat VERSION)"
|
||||
|
||||
- name: Build Docker image
|
||||
run: docker build -t obsigate:ci .
|
||||
- name: Configure DNS (workaround flaky 127.0.0.11 resolver)
|
||||
# GitHub Actions runners occasionally fail to resolve auth.docker.io via
|
||||
# the embedded Docker DNS (127.0.0.11:53 → "server misbehaving").
|
||||
# Force the daemon to use public DNS as a fallback.
|
||||
run: |
|
||||
sudo mkdir -p /etc/docker
|
||||
if ! grep -q "dns" /etc/docker/daemon.json 2>/dev/null; then
|
||||
echo '{"dns": ["8.8.8.8", "1.1.1.1", "9.9.9.9"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker || sudo service docker restart || true
|
||||
sleep 3
|
||||
fi
|
||||
|
||||
- name: Build Docker image (retry on transient DNS/network errors)
|
||||
run: |
|
||||
for attempt in 1 2 3; do
|
||||
echo "=== docker build attempt $attempt/3 ==="
|
||||
if docker build -t obsigate:ci . ; then
|
||||
echo "✓ Docker build succeeded"
|
||||
exit 0
|
||||
fi
|
||||
echo "✗ Build failed (attempt $attempt)"
|
||||
if [ $attempt -lt 3 ]; then
|
||||
sleep $((attempt * 10))
|
||||
fi
|
||||
done
|
||||
echo "✗ Docker build failed after 3 attempts"
|
||||
exit 1
|
||||
|
||||
- name: Verify image
|
||||
run: docker images obsigate:ci
|
||||
@@ -108,7 +252,7 @@ jobs:
|
||||
# ── E2E Tests (Playwright) ─────────────────────────────────────────
|
||||
e2e:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ci-deepin
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -122,10 +266,19 @@ jobs:
|
||||
npm ci
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
- name: Npm audit (bloquant — #87, 0 dépendance prod hors Playwright)
|
||||
run: npm audit --omit=dev
|
||||
|
||||
- name: Start ObsiGate
|
||||
run: |
|
||||
docker rm -f obsigate-e2e 2>/dev/null || true
|
||||
docker create --name obsigate-e2e -p 2029:8080 \
|
||||
# DinD robuste : PAS de port publié (les ports 2029 fuités par des
|
||||
# runs antérieurs font échouer « docker start », et l'IP passerelle
|
||||
# du job n'est pas toujours joignable → boucle de health curls sans
|
||||
# timeout qui pendaient 150 s chacun). À la place, l'app est
|
||||
# rattachée au réseau du job et jointable par nom DNS embarqué.
|
||||
JOBNET=$(docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$(hostname)" 2>/dev/null | awk '{print $1}')
|
||||
docker create --name obsigate-e2e \
|
||||
-v $(pwd)/test_vault:/vaults/TestVault \
|
||||
-v $(pwd)/test_dir:/vaults/TestDir \
|
||||
-e VAULT_1_NAME=TestVault \
|
||||
@@ -133,6 +286,7 @@ jobs:
|
||||
-e DIR_1_NAME=TestDir \
|
||||
-e DIR_1_PATH=/vaults/TestDir \
|
||||
-e OBSIGATE_AUTH_ENABLED=false \
|
||||
-e OBSIGATE_ALLOW_INSECURE=true \
|
||||
obsigate:ci
|
||||
# Docker-in-docker : le bind mount $(pwd)/... pointe sur un chemin
|
||||
# du job container, inexistant sur l'hôte → montage vide. Les -v
|
||||
@@ -140,23 +294,33 @@ jobs:
|
||||
# fixtures avant le démarrage (l'indexeur scanne au démarrage).
|
||||
docker cp test_vault/. obsigate-e2e:/vaults/TestVault
|
||||
docker cp test_dir/. obsigate-e2e:/vaults/TestDir
|
||||
if [ -n "$JOBNET" ]; then docker network connect "$JOBNET" obsigate-e2e || true; fi
|
||||
docker start obsigate-e2e
|
||||
# Le port publié est joignable via l'IP de la passerelle (localhost
|
||||
# du job container ne voit pas le port publié sur l'hôte).
|
||||
GW=$(ip route show default | awk '{print $3}' || echo 172.17.0.1)
|
||||
echo "Gateway IP: $GW"
|
||||
# Wait for health check
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://$GW:2029/api/health"; then echo "Health OK"; break; fi
|
||||
if [ -n "$JOBNET" ]; then
|
||||
APP_URL="http://obsigate-e2e:8080"
|
||||
else
|
||||
echo "Réseau du job introuvable (JOBNET vide) — impossible de localiser l'app sans port publié"
|
||||
exit 1
|
||||
fi
|
||||
echo "APP_URL: $APP_URL"
|
||||
# --max-time : ne JAMAIS pendre sans timeout (bug d'origine : curls
|
||||
# sans timeout = 150 s × 31 tentatives, job « bloqué »).
|
||||
for i in $(seq 1 40); do
|
||||
if curl -sf --max-time 3 "$APP_URL/api/health"; then echo "Health OK"; break; fi
|
||||
sleep 1
|
||||
done
|
||||
curl -sf "http://$GW:2029/api/health" >/dev/null || { echo "App not reachable at $GW:2029"; exit 1; }
|
||||
curl -sf --max-time 5 "$APP_URL/api/health" >/dev/null || { echo "App not reachable at $APP_URL"; docker logs --tail 40 obsigate-e2e; exit 1; }
|
||||
# Les fixtures sont copiées via `docker cp` en root → rendre le vault
|
||||
# inscriptible par l'utilisateur non-root de l'app (UID 1000), sinon
|
||||
# toute création/édition de fichier renvoie 403 « Vault is read-only ».
|
||||
docker exec -u 0 obsigate-e2e chmod -R a+rwX /vaults/TestVault /vaults/TestDir || true
|
||||
|
||||
- name: Run E2E tests
|
||||
run: |
|
||||
GW=$(ip route show default | awk '{print $3}' || echo 172.17.0.1)
|
||||
echo "Using BASE_URL=http://$GW:2029"
|
||||
BASE_URL="http://$GW:2029" npx playwright test --project=chromium-desktop --reporter=list
|
||||
BASE_URL="http://obsigate-e2e:8080"
|
||||
docker exec obsigate-e2e true >/dev/null 2>&1 || { echo "obsigate-e2e n'est pas en marche"; exit 1; }
|
||||
echo "Using BASE_URL=$BASE_URL"
|
||||
BASE_URL="$BASE_URL" npx playwright test --project=chromium-desktop --reporter=list
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
@@ -168,4 +332,4 @@ jobs:
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: docker rm -f obsigate-e2e
|
||||
run: docker rm -f obsigate-e2e
|
||||
@@ -1,13 +1,10 @@
|
||||
name: Desktop Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'desktop/**'
|
||||
- 'frontend/**'
|
||||
- 'backend/**'
|
||||
workflow_dispatch: # permet de lancer manuellement
|
||||
# Lancement manuel uniquement : aucun runner self-hosted [windows/linux, desktop]
|
||||
# n'est enregistré dans Gitea. Le build Windows se fait en local
|
||||
# (desktop/build-windows.bat). Ce workflow reste disponible pour un futur runner.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-windows:
|
||||
@@ -37,13 +34,27 @@ jobs:
|
||||
|
||||
- name: Build MSI
|
||||
working-directory: desktop
|
||||
run: cargo tauri build --bundles msi
|
||||
shell: powershell
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
run: |
|
||||
if ([string]::IsNullOrEmpty($env:TAURI_SIGNING_PRIVATE_KEY)) {
|
||||
Write-Host "TAURI_SIGNING_PRIVATE_KEY absent - build sans artefacts de mise a jour."
|
||||
cargo tauri build --bundles msi --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
} else {
|
||||
Write-Host "Signature des artefacts de mise a jour activee."
|
||||
cargo tauri build --bundles msi
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
|
||||
- name: Upload MSI artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: obsigate-windows-msi
|
||||
path: desktop/target/release/bundle/msi/*.msi
|
||||
path: |
|
||||
desktop/target/release/bundle/msi/*.msi
|
||||
desktop/target/release/bundle/msi/*.msi.sig
|
||||
retention-days: 30
|
||||
|
||||
- name: Publish to Gitea Release
|
||||
@@ -82,11 +93,31 @@ jobs:
|
||||
|
||||
- name: Build AppImage
|
||||
working-directory: desktop
|
||||
run: cargo tauri build --bundles appimage
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$TAURI_SIGNING_PRIVATE_KEY" ]; then
|
||||
echo "TAURI_SIGNING_PRIVATE_KEY absent - build sans artefacts de mise a jour."
|
||||
cargo tauri build --bundles appimage --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
else
|
||||
echo "Signature des artefacts de mise a jour activee."
|
||||
cargo tauri build --bundles appimage
|
||||
fi
|
||||
|
||||
- name: Build deb
|
||||
working-directory: desktop
|
||||
run: cargo tauri build --bundles deb
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$TAURI_SIGNING_PRIVATE_KEY" ]; then
|
||||
echo "TAURI_SIGNING_PRIVATE_KEY absent - build sans artefacts de mise a jour."
|
||||
cargo tauri build --bundles deb --config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
else
|
||||
echo "Signature des artefacts de mise a jour activee."
|
||||
cargo tauri build --bundles deb
|
||||
fi
|
||||
|
||||
- name: Upload Linux artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -94,5 +125,7 @@ jobs:
|
||||
name: obsigate-linux
|
||||
path: |
|
||||
desktop/target/release/bundle/appimage/*.AppImage
|
||||
desktop/target/release/bundle/appimage/*.AppImage.sig
|
||||
desktop/target/release/bundle/deb/*.deb
|
||||
desktop/target/release/bundle/deb/*.deb.sig
|
||||
retention-days: 30
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# -----------------------------------------------------------------------------
|
||||
# ObsiGate — rattache l'incrément de version au commit qui vient d'être créé,
|
||||
# puis crée le tag `vX.Y.Z` de la version livrée.
|
||||
#
|
||||
# Deux situations :
|
||||
# 1. `prepare-commit-msg` a incrémenté ./VERSION et mis à jour les fichiers
|
||||
# dérivés (package.json, desktop, READMEs, ROADMAP, CHANGELOG) : on les
|
||||
# rattache ici au commit via `--amend` (le commit n'est pas encore poussé) ;
|
||||
# 2. le message venait d'un éditeur : l'incrément a été différé, il est
|
||||
# calculé ici à partir du message final (`COMMIT_EDITMSG`).
|
||||
#
|
||||
# Résultat : la version livrée, le contenu du commit et le tag correspondent
|
||||
# toujours. Le tag est publié avec la branche à chaque push (`push.followTags`,
|
||||
# posé par scripts/install-hooks.sh).
|
||||
# -----------------------------------------------------------------------------
|
||||
set -eu
|
||||
|
||||
[ "${SKIP_VERSION_BUMP:-}" = "" ] || exit 0
|
||||
[ "${OBSIGATE_VERSION_AMENDING:-}" = "" ] || exit 0 # garde anti-récursion
|
||||
|
||||
root="$(git rev-parse --show-toplevel)"
|
||||
version_file="$root/VERSION"
|
||||
[ -f "$version_file" ] || exit 0
|
||||
|
||||
# Opérations en cours (rebase, merge, cherry-pick…) : ne jamais amender ici.
|
||||
for state in rebase-merge rebase-apply CHERRY_PICK_HEAD REVERT_HEAD MERGE_HEAD BISECT_LOG; do
|
||||
if [ -e "$(git rev-parse --git-path "$state")" ]; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
list="$(git rev-parse --git-path obsigate-version-files)"
|
||||
defer="$(git rev-parse --git-path obsigate-version-defer)"
|
||||
|
||||
# Cas 2 : l'incrément avait été différé (commit rédigé dans un éditeur).
|
||||
if [ -f "$defer" ]; then
|
||||
rm -f "$defer"
|
||||
rm -f "$list"
|
||||
tool="$root/scripts/bump_version.py"
|
||||
py=""
|
||||
for cand in python3 python; do
|
||||
if command -v "$cand" >/dev/null 2>&1; then py="$cand"; break; fi
|
||||
done
|
||||
if [ -n "$py" ] && [ -f "$tool" ]; then
|
||||
"$py" "$tool" --from-message-file "$(git rev-parse --git-path COMMIT_EDITMSG)" \
|
||||
--files-out "$list" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rattachement des fichiers synchronisés au commit qui vient d'être créé.
|
||||
if [ -s "$list" ]; then
|
||||
# shellcheck disable=SC2046
|
||||
git add -- $(tr -d '\r' < "$list")
|
||||
OBSIGATE_VERSION_AMENDING=1 \
|
||||
git commit --amend --no-edit --no-verify --quiet
|
||||
rm -f "$list"
|
||||
fi
|
||||
|
||||
ver="$(tr -d ' \t\r\n' < "$version_file")"
|
||||
[ -n "$ver" ] || exit 0
|
||||
|
||||
if git rev-parse -q --verify "refs/tags/v$ver" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git tag -a "v$ver" -m "ObsiGate v$ver"
|
||||
echo "ObsiGate version : tag v$ver créé (poussé avec la branche)."
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/bin/sh
|
||||
# -----------------------------------------------------------------------------
|
||||
# ObsiGate — incrément automatique de la version livrée (SemVer) à chaque commit.
|
||||
#
|
||||
# Source unique de vérité : ./VERSION (MAJEUR.MINEUR.CORRECTIF)
|
||||
#
|
||||
# `!:` / `BREAKING CHANGE:` -> MAJEUR (x.0.0)
|
||||
# `feat:` -> MINEUR (x.y.0)
|
||||
# tout le reste (fix, perf…) -> CORRECTIF (x.y.z)
|
||||
#
|
||||
# Aucun incrément pour : merge, squash, revert, `chore(release)`, amend.
|
||||
# Commit rédigé dans un éditeur (message pas encore saisi) : l'incrément est
|
||||
# différé à `post-commit`, qui dispose du message final.
|
||||
#
|
||||
# `scripts/bump_version.py` incrémente VERSION et resynchronise les fichiers
|
||||
# dérivés (package.json, desktop, READMEs, ROADMAP, CHANGELOG) ; le `git add` est
|
||||
# fait ici, par le hook lui-même. La liste des fichiers est laissée à
|
||||
# `.githooks/post-commit`, qui rattache l'incrément au commit (git fige l'arbre
|
||||
# avant `prepare-commit-msg` : un `git add` à cet instant ne serait pris en
|
||||
# compte qu'au commit suivant).
|
||||
#
|
||||
# Neutraliser ponctuellement : SKIP_VERSION_BUMP=1 git commit ...
|
||||
# -----------------------------------------------------------------------------
|
||||
set -eu
|
||||
|
||||
[ "${SKIP_VERSION_BUMP:-}" = "" ] || exit 0
|
||||
|
||||
msg_file="${1:-}"
|
||||
source_kind="${2:-message}"
|
||||
|
||||
case "$source_kind" in
|
||||
merge|squash|commit) exit 0 ;; # merge / squash / amend (-c, -C, --amend)
|
||||
esac
|
||||
|
||||
[ -n "$msg_file" ] && [ -f "$msg_file" ] || exit 0
|
||||
|
||||
root="$(git rev-parse --show-toplevel)"
|
||||
tool="$root/scripts/bump_version.py"
|
||||
[ -f "$tool" ] || exit 0
|
||||
|
||||
py=""
|
||||
for cand in python3 python; do
|
||||
if command -v "$cand" >/dev/null 2>&1; then py="$cand"; break; fi
|
||||
done
|
||||
if [ -z "$py" ]; then
|
||||
echo "⚠ version : aucun interpréteur Python trouvé — version non incrémentée." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
list="$(git rev-parse --git-path obsigate-version-files)"
|
||||
defer="$(git rev-parse --git-path obsigate-version-defer)"
|
||||
|
||||
# Message pas encore rédigé (ouverture d'un éditeur) : on diffère l'incrément.
|
||||
if ! grep -qvE '^[[:space:]]*(#|$)' "$msg_file"; then
|
||||
: > "$defer"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -f "$defer" # marqueur d'un commit précédent avorté
|
||||
|
||||
out=""
|
||||
if out="$("$py" "$tool" --from-message-file "$msg_file" --files-out "$list" 2>&1)"; then
|
||||
printf '%s\n' "$out"
|
||||
else
|
||||
status=$?
|
||||
if [ "$status" = "3" ]; then
|
||||
rm -f "$list"
|
||||
exit 0 # 3 = version inchangée (revert, release…) : pas une erreur
|
||||
fi
|
||||
echo "✗ version : échec de l'incrément automatique de ./VERSION" >&2
|
||||
printf '%s\n' "$out" >&2
|
||||
echo " → corriger, ou passer outre avec : SKIP_VERSION_BUMP=1 git commit ..." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ajout des fichiers synchronisés (chemins explicites).
|
||||
if [ -s "$list" ]; then
|
||||
# shellcheck disable=SC2046
|
||||
git add -- $(tr -d '\r' < "$list")
|
||||
fi
|
||||
+24
@@ -26,5 +26,29 @@ playwright-report/
|
||||
desktop/target/
|
||||
desktop/python-embed/
|
||||
desktop/frontend-static/
|
||||
desktop/checksums.txt
|
||||
desktop/backend/
|
||||
desktop/frontend/
|
||||
backend/VERSION
|
||||
|
||||
# Artefacts générés par les runs E2E (excalidraw crée ces diagrammes)
|
||||
test_vault/IT/e2e-diagram-*.excalidraw
|
||||
|
||||
# Fixtures de test locales non versionnées (~200 Mo, pas de fixture CI).
|
||||
# Aucun test/CI ne les référence : les tests unitaires génèrent leurs fixtures
|
||||
# dans tmp_path (tests/conftest.py), et l'E2E n'utilise que les fixtures
|
||||
# committées (test_vault/sample-*.{mp3,png,svg,webm,pdf}, test_dir/*.md).
|
||||
# → à committer volontairement : `git add -f <chemin>`.
|
||||
test_dir/music/
|
||||
test_dir/video/
|
||||
test_vault/images/
|
||||
test_vault/markdown/
|
||||
test_vault/budget.xlsx
|
||||
test_home/
|
||||
|
||||
# Tauri updater signing keys (private key — never commit)
|
||||
desktop/*.key
|
||||
desktop/*.key.pub
|
||||
desktop/key/
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# AGENTS.md — Instructions obligatoires du dépôt ObsiGate
|
||||
|
||||
> Ces instructions s'appliquent à **toute** intervention (humaine ou IA) sur ce dépôt.
|
||||
> Documentation et réponses en **français**.
|
||||
|
||||
## Règle n°1 — Méthode de livraison unique
|
||||
|
||||
Avant toute tâche (fonctionnalité, bug, refactor), **lire et appliquer**
|
||||
[`docs/DELIVERY_WORKFLOW.md`](./docs/DELIVERY_WORKFLOW.md) (Definition of Done).
|
||||
Aucune tâche n'est terminée avant que sa checklist soit complète **et le CI vert**
|
||||
(jobs `lint`, `test`, `security`, `build`, `e2e` de `.gitea/workflows/ci.yml`).
|
||||
|
||||
## Avant de commencer
|
||||
|
||||
1. Lire [`docs/ROADMAP.md`](./docs/ROADMAP.md) (travail à venir + index) et
|
||||
[`docs/ISSUES_TODOLIST.md`](./docs/ISSUES_TODOLIST.md) (bugs).
|
||||
2. Identifier ou créer l'**ID stable** (`#NN` pour une feature, `BUG-NNN` pour un bug —
|
||||
jamais réutilisé) et passer son statut à « en cours » **avant** de coder.
|
||||
|
||||
## Architecture (ce qui n'est pas obvious)
|
||||
|
||||
- **Backend** : FastAPI/Python 3.11, point d'entrée `backend/main.py` (endpoints + rendu
|
||||
markdown), index en mémoire (`indexer.py`, `search.py`), watcher (`watcher.py`),
|
||||
auth dans `backend/auth/`. Pas de base de données : JSON dans `data/`.
|
||||
- **Frontend** : vanilla JS **zéro framework, zéro build npm** (`frontend/app.js`,
|
||||
`index.html`, `style.css`). Ne pas ajouter de dépendances npm ni d'étape de build.
|
||||
- **Desktop** : Tauri (Rust) dans `desktop/` ; `tauri.conf.json` embarque `backend/**` et
|
||||
`frontend/**` depuis `desktop/` — les scripts de build font le **staging** (copie) avant
|
||||
`cargo tauri build`, sinon le build échoue.
|
||||
- **i18n** : tout texte d'interface doit exister en FR **et** EN
|
||||
(`frontend/locales/fr.json` + `en.json`).
|
||||
|
||||
## Vérifications locales (pwsh, à faire passer avant tout commit/push)
|
||||
|
||||
```powershell
|
||||
# Backend (venv à la racine)
|
||||
.\.venv\Scripts\python.exe -m pytest tests/
|
||||
.\.venv\Scripts\python.exe -m ruff check backend/
|
||||
.\.venv\Scripts\python.exe -m mypy backend/ --ignore-missing-imports
|
||||
|
||||
# Frontend : scripts Node à exécuter directement (pas de runner)
|
||||
node tests/frontend/validate-imports.mjs
|
||||
node tests/frontend/unit.test.mjs
|
||||
# Tests JSDOM : node_modules dans tests/frontend/ (npm install là-bas si absent), ex :
|
||||
node tests/frontend/pane-manager.test.mjs
|
||||
|
||||
# E2E (si UI touchée, ~10 min) : reproduit le job CI e2e (port 2029, auth désactivée)
|
||||
npm run test:e2e # prérequis : uv, Node >= 20, npx playwright install chromium
|
||||
bash scripts/run-e2e-local.sh -g "nom du test" # filtre / --headed
|
||||
|
||||
# Windows sans bash exploitable (WSL HS, git-bash bloqué par App Control) :
|
||||
npm run test:e2e:ps # équivalent PowerShell, mêmes conditions que le CI
|
||||
pwsh -File scripts/run-e2e-local.ps1 -PlaywrightArgs @('-g','nom du test')
|
||||
```
|
||||
|
||||
- Un seul test backend : `.\.venv\Scripts\python.exe -m pytest tests/test_search.py -q`.
|
||||
- **Sélection E2E** : vérifier chaque sélecteur dans le DOM réel avant de l'utiliser dans un
|
||||
test ; tout test nouveau/modifié doit passer en local avant push ; pas de contournement
|
||||
qui masque la flakiness (`waitForTimeout` arbitraires, fallbacks silencieux).
|
||||
- La suite E2E doit finir à **100 %** sans s'appuyer sur les retries. Jamais de `git push`
|
||||
avant que les 5 étapes locales soient vertes.
|
||||
|
||||
## Version & hooks (pièges)
|
||||
|
||||
- `VERSION` (racine) = **source unique de vérité** (SemVer), incrémenté **automatiquement à
|
||||
chaque commit** par le hook `.githooks/prepare-commit-msg` — `feat` → mineur,
|
||||
`!:` / `BREAKING CHANGE` → majeur, sinon correctif. Le même commit resynchronise
|
||||
`package.json`, le desktop Tauri, `README.md`/`README.fr.md`, `docs/ROADMAP.md` et publie
|
||||
la section `[Unreleased]` du `CHANGELOG.md` en `[X.Y.Z] — date` ; tag `vX.Y.Z` créé au
|
||||
commit, publié au push (`push.followTags`).
|
||||
- Hooks **obligatoires**, à installer une fois par clone : `scripts/install-hooks.sh`
|
||||
(sinon la version ne suit plus et le CI échoue via le garde-fou `tests/test_version.py`).
|
||||
- Le rattachement des fichiers de bump se fait par un `--amend` immédiat : **le SHA affiché
|
||||
par `git commit` change** — ne pas s'y fier.
|
||||
- Commit sans incrément (exceptionnel) : `SKIP_VERSION_BUMP=1 git commit …`.
|
||||
- Ne jamais réécrire une version déjà publiée dans le CHANGELOG ; jamais de détail dupliqué
|
||||
entre Roadmap et CHANGELOG.
|
||||
|
||||
## À la fin de chaque tâche (obligatoire)
|
||||
|
||||
- Tests unitaires ajoutés/mis à jour (correctif sans test de non-régression = pas terminé).
|
||||
- Toutes les vérifications locales ci-dessus vertes (`E2E` si UI).
|
||||
- Documentation mise à jour : `CHANGELOG.md` (`[Unreleased]`), `docs/ROADMAP.md` (statut +
|
||||
index), fiche `docs/features/` **ou** `docs/archive/`, `docs/ISSUES_TODOLIST.md` (si bug),
|
||||
guide utilisateur i18n FR/EN + README si impact utilisateur, docstrings +
|
||||
`response_model` si API.
|
||||
- **Commit** conventionnel référençant l'ID (`feat: … #12`), puis **push** et **CI vert**.
|
||||
|
||||
## Cartographie documentaire
|
||||
|
||||
| Sujet | Fichier |
|
||||
|---|---|
|
||||
| Méthode de livraison / DoD | `docs/DELIVERY_WORKFLOW.md` |
|
||||
| Version livrée (source unique) | `VERSION` + `scripts/bump_version.py` |
|
||||
| Travail à venir + index | `docs/ROADMAP.md` |
|
||||
| Historique des versions | `CHANGELOG.md` |
|
||||
| Conception par feature | `docs/features/<slug>.md` |
|
||||
| Guides d'utilisation | `docs/GUIDES/` |
|
||||
| Archive du complété | `docs/archive/COMPLETED_v1-v2.md` |
|
||||
| Bugs / TODO | `docs/ISSUES_TODOLIST.md` |
|
||||
| Build & releases | `docs/DEVELOPMENT_AND_RELEASES.md` |
|
||||
| Standards de code | `docs/CONTRIBUTING.md` |
|
||||
|
||||
## Conventions
|
||||
|
||||
- Commits : `type: description` — `feat`, `fix`, `perf`, `refactor`, `docs`, `style`, `chore`, `test`.
|
||||
- Sécurité : tout chemin fichier fourni par l'utilisateur passe par `_resolve_safe_path()`.
|
||||
- **Ne jamais** committer de secrets, clés ou tokens (`.env` jamais committé ; secrets dans
|
||||
`data/api_keys.json` ou variables `OBSIGATE_*`).
|
||||
- Respecter le style du code existant (ruff/mypy 0 erreur ; CSS variables, pas de couleurs
|
||||
hardcodées ; `safeCreateIcons()` plutôt que `lucide.createIcons()` direct).
|
||||
+4681
-3
File diff suppressed because it is too large
Load Diff
+9
-5
@@ -3,7 +3,8 @@
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends gcc libffi-dev libc6-dev libpango-1.0-0 libpangocairo-1.0-0 shared-mime-info \
|
||||
&& apt-get install -y --no-install-recommends gcc libffi-dev libc6-dev \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
@@ -14,7 +15,6 @@ RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||
FROM python:3.11-slim
|
||||
|
||||
LABEL maintainer="Bruno Beloeil" \
|
||||
version="1.4.0" \
|
||||
description="ObsiGate — lightweight web interface for Obsidian vaults"
|
||||
|
||||
WORKDIR /app
|
||||
@@ -24,20 +24,24 @@ COPY --from=builder /install /usr/local
|
||||
|
||||
# WeasyPrint runtime dependencies
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libpango-1.0-0 libpangocairo-1.0-0 shared-mime-info \
|
||||
&& apt-get install -y --no-install-recommends libpango-1.0-0 libpangocairo-1.0-0 shared-mime-info fonts-noto-color-emoji \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy application code
|
||||
COPY backend/ ./backend/
|
||||
COPY frontend/ ./frontend/
|
||||
|
||||
# Copy pre-generated VERSION file (generated by build.sh before docker build)
|
||||
COPY backend/VERSION ./backend/VERSION
|
||||
# Version livrée : `VERSION` (racine du dépôt) est la source unique de vérité —
|
||||
# copié dans l'image, jamais un numéro codé en dur. `backend/version.py` le lit
|
||||
# et /api/health l'expose (header + boîte À propos).
|
||||
COPY VERSION ./VERSION
|
||||
|
||||
# Create non-root user for security + data directory for auth persistence
|
||||
# Using explicit UID/GID 1000 to match common host user and docker-compose settings
|
||||
RUN groupadd -g 1000 obsigate && useradd -u 1000 -g obsigate -d /app -s /sbin/nologin obsigate \
|
||||
&& mkdir -p /app/data \
|
||||
&& chmod 777 /app/data \
|
||||
&& chown -R obsigate:obsigate /app
|
||||
USER obsigate
|
||||
|
||||
|
||||
+157
-44
@@ -1,70 +1,91 @@
|
||||
# ObsiGate
|
||||
|
||||
> **Version française** — ce document est le miroir synchronisé de [README.md](README.md) (référence complète). Dernière synchronisation : juin 2026.
|
||||
> **Version française** — ce document est le miroir synchronisé de [README.md](README.md) (référence complète). Dernière synchronisation : septembre 2026.
|
||||
|
||||
**Porte d'entrée web ultra-léger pour vos vaults Obsidian** — Accédez, naviguez et recherchez dans toutes vos notes Obsidian depuis n'importe quel appareil via une interface web moderne et responsive.
|
||||
|
||||
[]()
|
||||
[]()
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://www.docker.com/)
|
||||
[](https://www.python.org/)
|
||||
[](https://git.dracodev.net/Projets/ObsiGate/actions)
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ [🔍 Recherche...] [☀/🌙 Thème] ObsiGate │
|
||||
├──────────────┬──────────────────────────────────────────┤
|
||||
│ SIDEBAR │ CONTENT AREA │
|
||||
│ ▼ Recettes │ 📄 Titre du fichier │
|
||||
│ 📁 Soupes │ Tags: #recette #rapide │
|
||||
│ 📄 Pizza │ [Contenu Markdown rendu] │
|
||||
│ ▼ IT │ │
|
||||
│ 📁 Docker │ │
|
||||
│ Tags Cloud │ │
|
||||
└──────────────┴──────────────────────────────────────────┘
|
||||
```
|
||||

|
||||
|
||||
> Interface web d'ObsiGate : sidebar multi-vault, recherche globale, statistiques et raccourcis.
|
||||
|
||||
---
|
||||
|
||||
## 📚 Guides
|
||||
|
||||
Les **guides d'utilisation** pas à pas se trouvent dans [`docs/GUIDES/`](docs/GUIDES/) :
|
||||
|
||||
| Guide | Contenu |
|
||||
|---|---|
|
||||
| 🚀 [Prise en main](docs/GUIDES/PRISE_EN_MAIN.md) | Premier lancement, interface, navigation, vaults, raccourcis |
|
||||
| 🔍 [Recherche, PDF, Excel & Excalidraw](docs/GUIDES/RECHERCHE_PDF_EXCALIDRAW.md) | Syntaxe de requête, recherche sémantique, lecteurs PDF/Excel, diagrammes |
|
||||
| 🤖 [Assistant IA & Forge](docs/GUIDES/ASSISTANT_IA_FORGE.md) | Fournisseurs, éditeur IA, BooksLM, Forge, commandes `@` / `/` |
|
||||
| 📝 [Édition & collaboration](docs/GUIDES/COLLABORATION.md) | Édition simultanée, curseurs distants, persistance |
|
||||
| 📱 [PWA & hors-ligne](docs/GUIDES/PWA_HORS_LIGNE.md) | Installation, cache hors-ligne, file de synchro, notifications |
|
||||
| 🔌 [API REST](docs/GUIDES/API_REST.md) | Authentification, clés API, endpoints, exemples `curl`, SSE |
|
||||
| 🧩 [Serveur MCP](docs/GUIDES/MCP.md) | Brancher Claude Desktop, Cursor, Cline… sur vos vaults |
|
||||
| 🔒 [Authentification & sécurité](docs/GUIDES/AUTHENTIFICATION_SECURITE.md) | Utilisateurs, MFA, permissions par vault, durcissement |
|
||||
| 🐳 [Déploiement Docker](docs/GUIDES/DEPLOIEMENT_DOCKER.md) | `docker-compose`, volumes, reverse proxy, mises à jour |
|
||||
| 🖥️ [Desktop (Tauri)](docs/GUIDES/DESKTOP.md) | Installation, premier lancement, build depuis les sources, dépannage |
|
||||
|
||||
> Index complet : [`docs/GUIDES/README.md`](docs/GUIDES/README.md).
|
||||
|
||||
---
|
||||
|
||||
## 📋 Table des matières
|
||||
|
||||
- [Fonctionnalités](#fonctionnalites)
|
||||
- [Prérequis](#prerequis)
|
||||
- [Installation rapide](#installation-rapide)
|
||||
- [Configuration détaillée](#configuration-detaillee)
|
||||
- [Variables d'environnement](#variables-denvironnement)
|
||||
- [🔒 Authentification](#authentification)
|
||||
- [Ajouter une nouvelle vault](#ajouter-une-nouvelle-vault)
|
||||
- [Build & déploiement avec build.sh](#build-deploiement-avec-buildsh)
|
||||
- [Rendu d'images Obsidian](#rendu-dimages-obsidian)
|
||||
- [Desktop (Tauri) — Application native](#desktop-tauri-application-native)
|
||||
- [Utilisation](#utilisation)
|
||||
- [API](#api)
|
||||
- [Recherche avancée](#recherche-avancee)
|
||||
- [Dépannage](#depannage)
|
||||
- [Performance](#performance)
|
||||
- [Sécurité](#securite)
|
||||
- [Stack technique](#stack-technique)
|
||||
- [Architecture](#architecture)
|
||||
- [Développement](#developpement)
|
||||
- [Licence](#licence)
|
||||
- [Changelog](#changelog)
|
||||
- ✨ [Fonctionnalités](#fonctionnalites)
|
||||
- 📚 [Guides](#guides)
|
||||
- 🚀 [Prérequis](#prerequis)
|
||||
- ⚡ [Installation rapide](#installation-rapide)
|
||||
- ⚙️ [Configuration détaillée](#configuration-detaillee)
|
||||
- 🌍 [Variables d'environnement](#variables-denvironnement)
|
||||
- 🔒 [Authentification](#authentification)
|
||||
- ➕ [Ajouter une nouvelle vault](#ajouter-une-nouvelle-vault)
|
||||
- 🔨 [Build & déploiement avec build.sh](#build-deploiement-avec-buildsh)
|
||||
- 🖼️ [Rendu d'images Obsidian](#rendu-dimages-obsidian)
|
||||
- 🖥️ [Desktop (Tauri) — Application native](#desktop-tauri-application-native)
|
||||
- 📖 [Utilisation](#utilisation)
|
||||
- 👥 [Collaboration temps réel](#collaboration-temps-reel)
|
||||
- 🔌 [API](#api)
|
||||
- 🔍 [Recherche avancée](#recherche-avancee)
|
||||
- 🔧 [Dépannage](#depannage)
|
||||
- ⚡ [Performance](#performance)
|
||||
- 🛡️ [Sécurité](#securite)
|
||||
- 🏗️ [Stack technique](#stack-technique)
|
||||
- 🏠 [Architecture](#architecture)
|
||||
- 📝 [Développement](#developpement)
|
||||
- 📄 [Licence](#licence)
|
||||
- 🤝 [Support](#support)
|
||||
- 📝 [Changelog](#changelog)
|
||||
|
||||
---
|
||||
|
||||
## ✨ Fonctionnalités
|
||||
|
||||
- **🤖 AI Editor intégré** — Éditeur CodeMirror 6 avec toolbar IA : amélioration, correction, traduction, génération, réécriture personnalisée, toolbox (liste, tableau, frontmatter, canvas) — multi-provider DeepSeek/OpenRouter/Gemini
|
||||
- **🧩 Serveur MCP & agent IA** — Serveur Model Context Protocol intégré (`/mcp`) et assistant avec function calling : lisez, cherchez et modifiez vos vaults depuis Claude Desktop, Cursor… avec confirmations two-step, permissions par vault, rate limiting et redaction des secrets ([guide](docs/GUIDES/MCP.md))
|
||||
- **👥 Collaboration temps réel** — Édition simultanée d'un même document (Yjs/CRDT) : curseurs distants colorés, indicateur de présence, fusion sans conflit, reconnexion automatique et persistance serveur ([détail](docs/features/collaboration.md))
|
||||
- **📖 Guide d'utilisation intégré** — Aide complète en FR/EN accessible depuis le menu Options : interface, navigation, recherche, fichiers, IA, sécurité, API & intégrations (OpenAPI, MCP), hors-ligne, collaboration, desktop, plus une section **Architecture** avec diagramme Mermaid ; téléchargeable en **Markdown** et **PDF** dans la langue courante ([détail](docs/features/guide-coverage-105.md))
|
||||
- **📱 Éditeur mobile natif** — Édition optimisée pour le tactile : barre d'outils Markdown flottante (gras/italique/code/liste/lien), bouton « Coller » persistant (contournement iOS), zoom par pincement et hauteur ajustable, raccourcis swipe (liens entrants / table des matières) et mode lecture plein écran avec navigation entre fichiers ([détail](docs/features/mobile-editor.md))
|
||||
- **🗺️ Vue graphe interactive** — Canvas force-directed avec Barnes-Hut O(n log n), filtres (tag, type), profondeur, mode focus, historique de navigation ←→↑, export PNG, aperçu au survol (Ctrl+click)
|
||||
- **🗂️ Multi-vault** : Visualisez plusieurs vaults Obsidian simultanément
|
||||
- **🌳 Navigation arborescente** : Parcourez vos dossiers et fichiers dans la sidebar
|
||||
- **🔍 Recherche avancée** : Moteur TF-IDF avec stemming français, normalisation des accents, snippets surlignés, facettes, pagination et tri
|
||||
- **🌳 Navigation arborescente** : Parcourez vos dossiers et fichiers dans la sidebar ; chaque clic sur un répertoire de l'arbre ouvre un **onglet de navigation** — chemin, récents, sous-répertoires cliquables, facettes Vaults · Tags · Extensions, tri Pertinence/Date et enregistrement du répertoire — qui coexiste avec vos fichiers ouverts
|
||||
- **🔍 Recherche avancée** : Moteur TF-IDF avec stemming français, normalisation des accents, snippets surlignés, facettes, pagination et tri — plus une **recherche sémantique** optionnelle (embeddings `all-MiniLM-L6-v2`, fusion hybride TF-IDF + RRF) activable via le toggle `~` ([détail](docs/features/semantic-search.md))
|
||||
- **💡 Autocomplétion intelligente** : Suggestions de fichiers, tags et historique avec navigation clavier
|
||||
- **🧩 Syntaxe de requête** : Opérateurs `tag:`, `#`, `vault:`, `title:`, `path:`, `ext:` avec chips visuels
|
||||
- **📜 Historique de recherche** : Persisté en localStorage (max 50 entrées, LIFO, dédupliqué)
|
||||
- **🏷️ Tag cloud** : Filtrage par tags extraits des frontmatters YAML
|
||||
- **🔗 Wikilinks** : Les `[[liens internes]]` Obsidian sont cliquables
|
||||
- **🖼️ Images Obsidian** : Support complet des syntaxes d'images Obsidian avec résolution intelligente
|
||||
- **🎬 Audio & vidéo** : Lecteurs HTML5 intégrés (`.mp3 .wav .flac .mp4 .webm`…) avec streaming HTTP Range (lecture, déplacement, plein écran) et **lecture persistante** (mini-lecteur flottant / mini-fenêtre vidéo, retour au média ou arrêt à tout moment, contrôles écran verrouillé via Media Session), repli téléchargement si le format n'est pas lisible par le navigateur
|
||||
- **🎨 Diagrammes Excalidraw** : Visualiseur/éditeur natif des fichiers `.excalidraw` et `.excalidraw.md` (iframe sandboxée, auto-save, thème clair/sombre, texte des diagrammes indexé pour la recherche)
|
||||
- **📊 Tableurs Excel** : les fichiers `.xlsx` et `.xlsm` s'ouvrent dans un visualiseur dédié — un tableau par feuille avec onglets, en-têtes A1 et édition directe des cellules (`PUT /api/file/{vault}/xlsx/save`, backup automatique, écriture atomique), plus le téléchargement du fichier d'origine. Le visualiseur rend polices, couleurs, cellules fusionnées et volets figés, et offre navigation et raccourcis clavier (`Ctrl+S`, `Suppr`, `F2`, `Ctrl+Origine/Fin`, `PgPréc/PgSuiv`, `Ctrl+flèches`), barre de formule avec noms de fonctions, zone Nom éditable (« Atteindre » `A1:B3`), presse-papiers de plage (copier/couper/coller un bloc, depuis ou vers Excel), un menu **Mise en forme** (gras/italique/souligné, alignements, couleurs, formats de nombre, fusions, volets figés, largeur/hauteur — `PUT /api/file/{vault}/xlsx/style`), tri/filtre/recherche sur toutes les feuilles, export CSV/Markdown/HTML et impression (sélection ou feuille), édition de la structure (feuilles, lignes, colonnes) et un tableau de bord du classeur (plages nommées, détection graphiques/TCD, stats par feuille) ; un `.csv` s'édite dans la même grille (RFC 4180) tandis que `.xls` et `.ods` s'ouvrent en lecture seule. Les classeurs contenant des éléments qu'ObsiGate ne peut pas conserver (valeurs calculées, segments, contrôles de formulaire, signature…) affichent un **avertissement** et demandent confirmation avant l'enregistrement ; une saisie commençant par `=` ou `@` est stockée comme texte sauf activation du bouton `f(x)`, et les écritures concurrentes d'un autre poste sont détectées (`If-Match` → « Réessayer »). L'assistant IA peut lister les feuilles, injecter un tableau borné dans son contexte, rechercher dans le classeur, analyser une plage, modifier des cellules et ajouter des lignes — sur `.xlsx`, `.xlsm` et `.csv`. Sur mobile (≤ 768 px), la barre de menus et le ruban sont **repliés par défaut** — un bouton ☰ les déplie — pour que la grille occupe toute la hauteur d'écran
|
||||
- **🎨 Syntax highlight** : Coloration syntaxique des blocs de code
|
||||
- **🌓 Thème clair/sombre** : Toggle persisté en localStorage
|
||||
- **📡 Synchronisation temps réel** : Surveillance automatique des fichiers via watchdog avec mise à jour incrémentale de l'index
|
||||
@@ -201,6 +222,7 @@ Les vaults sont configurées par paires `VAULT_N_NAME` / `VAULT_N_PATH` (N = 1,
|
||||
| `VAULT_1_PATH` | Chemin dans le conteneur | `/vaults/Obsidian-RECETTES` |
|
||||
| `VAULT_1_ATTACHMENTS_PATH` | Dossier d'attachements (optionnel) | `06_Boite_a_Outils/6.2_Attachments` |
|
||||
| `VAULT_1_SCAN_ATTACHMENTS` | Scan d'images au démarrage (défaut : true) | `true` |
|
||||
| `OBSIGATE_HOME_ROOT` | Racine des dossiers personnels (un vault `home-<user>` par compte). Absente = fonctionnalité désactivée. | `/vaults/Home` |
|
||||
|
||||
**Règles de nommage :** lettres, chiffres et tirets uniquement ; pas d'espaces ; le nom doit correspondre au chemin dans le conteneur.
|
||||
|
||||
@@ -271,7 +293,24 @@ Un compte **admin** connecté voit une icône 🛡️ dans le header : liste, cr
|
||||
| `OBSIGATE_ACCESS_TOKEN_TTL` | Durée de vie token JWT (secondes) | `3600` |
|
||||
| `OBSIGATE_REFRESH_TOKEN_TTL` | Durée de vie refresh token (secondes) | `2592000` |
|
||||
| `OBSIGATE_LOGIN_MAX_ATTEMPTS` | Tentatives de login max par IP | `10` |
|
||||
| `OBSIGATE_ACCOUNT_MAX_ATTEMPTS` | Tentatives de login max par compte | `10` |
|
||||
| `OBSIGATE_LOGIN_WINDOW_SECONDS` | Fenêtre de rate limiting (secondes) | `900` |
|
||||
| `OBSIGATE_TRUST_PROXY` | Faire confiance à `X-Forwarded-For` pour l'IP client (reverse proxy) | `false` |
|
||||
| `OBSIGATE_WEBHOOK_ALLOW_HTTP` | Autoriser les webhooks non HTTPS | `false` |
|
||||
| `OBSIGATE_WEBHOOK_ALLOW_PRIVATE` | Autoriser les webhooks vers des adresses privées/boucle | `false` |
|
||||
| `OBSIGATE_PDF_MAX_SIZE_MB` | Taille max des PDF extraits (text indexation) | `50` |
|
||||
| `OBSIGATE_MEDIA_MAX_INLINE_MB` | Taille max pour la lecture audio/vidéo intégrée (au-delà : téléchargement) | `500` |
|
||||
| `OBSIGATE_PDF_EXTRACT_TIMEOUT` | Timeout extraction PDF (secondes) | `30` |
|
||||
| `OBSIGATE_TAVILY_API_KEY` / `OBSIGATE_BRAVE_API_KEY` / `OBSIGATE_SERPAPI_API_KEY` / `OBSIGATE_EXA_API_KEY` | Fournisseurs de recherche web à clé (essayés avant SearXNG) | — |
|
||||
| `OBSIGATE_WEB_PROVIDERS` | Ordre des fournisseurs de recherche (ex. `brave,searxng`) | — |
|
||||
| `OBSIGATE_WEB_RETRY` | Réessais réseau des outils web (backoff maison) | `1` |
|
||||
| `OBSIGATE_WEB_CACHE_TTL` | Durée du cache SQLite des résultats web (secondes, `0` = off) | `900` |
|
||||
| `OBSIGATE_GITEA_URL` / `OBSIGATE_GITEA_TOKEN` | Source connectée Gitea (outil `git_list_repos`…) | — |
|
||||
| `OBSIGATE_GITHUB_TOKEN` | Jeton GitHub (outil `git_list_repos`…) | — |
|
||||
|
||||
> Ces clés peuvent aussi être saisies **depuis l'interface** (menu → Configurations →
|
||||
> « Sources connectées & recherche ») : la valeur saisie est stockée dans `data/api_keys.json`
|
||||
> et prime sur la variable d'environnement.
|
||||
|
||||
### Volume pour la persistance
|
||||
|
||||
@@ -371,6 +410,18 @@ ObsiGate supporte **toutes les syntaxes d'images Obsidian** avec résolution int
|
||||
6. Index de démarrage (match le plus proche)
|
||||
7. Fallback : placeholder stylisé `[image not found: filename.ext]`
|
||||
|
||||
### Visionneuse & arborescence
|
||||
|
||||
Les images sont de plein droit des fichiers du vault : elles apparaissent dans
|
||||
l'arborescence, sont indexées (nom + métadonnées, **jamais les octets**) et
|
||||
s'ouvrent dans une **visionneuse dédiée** — zoom molette 0,1×–8×, pan au
|
||||
glisser, double-clic pour réinitialiser, navigation ←/→ entre les images du
|
||||
dossier (avec pellicule de miniatures WebP), panneau de métadonnées, lightbox
|
||||
plein écran, ouverture de l'original et téléchargement. Le filtre de recherche
|
||||
`ext:png`/`ext:jpg` est disponible. Formats décodables : PNG, JPEG, GIF, WebP,
|
||||
BMP, ICO, SVG (SVG servi avec une politique CSP `sandbox`). **HEIC/HEIF**
|
||||
(iPhone) n'est pas décodable par les navigateurs et n'est pas pris en charge.
|
||||
|
||||
### Configuration
|
||||
|
||||
```yaml
|
||||
@@ -391,6 +442,8 @@ curl -X POST http://localhost:2020/api/attachments/rescan/MonVault
|
||||
|
||||
## 🖥️ Desktop (Tauri) — Application native
|
||||
|
||||
> 📖 Guide complet : [Desktop (Tauri)](docs/GUIDES/DESKTOP.md)
|
||||
|
||||
ObsiGate Desktop est une application native construite avec [Tauri](https://tauri.app/) (Rust + webview système). Elle embarque le backend Python et le frontend dans un exécutable standalone — zéro Docker, zéro ligne de commande.
|
||||
|
||||
> 🚧 **Version 2.0.0 — binaires en cours de stabilisation.** Pour l'instant, le build depuis les sources est recommandé.
|
||||
@@ -548,8 +601,31 @@ Cycle de vie : Tauri spawn le backend Python → health check → splash de dém
|
||||
|
||||
---
|
||||
|
||||
## 👥 Collaboration temps réel
|
||||
|
||||
> 📖 Guide complet : [Édition & collaboration](docs/GUIDES/COLLABORATION.md)
|
||||
|
||||
Plusieurs utilisateurs peuvent éditer le même document markdown simultanément (façon Google Docs) :
|
||||
|
||||
- **Fusion sans conflit** grâce à Yjs (CRDT) : deux personnes peuvent taper au même endroit, aucune
|
||||
modification n'est perdue.
|
||||
- **Curseurs distants colorés** et sélections visibles dans CodeMirror, avec le nom de chaque
|
||||
utilisateur.
|
||||
- **Indicateur de présence** dans l'en-tête de l'éditeur (avatars + statut de connexion).
|
||||
- **Reconnexion automatique** (backoff exponentiel) : l'état est fusionné au retour.
|
||||
- **Persistance serveur** : le document est écrit sur disque 2 s après la dernière modification.
|
||||
- **Transport** : WebSocket `ws(s)://<hôte>/ws/collab/{vault}/{chemin}`, authentifié par cookie
|
||||
`access_token` (ou `?token=`) et soumis au contrôle d'accès par vault.
|
||||
|
||||
Aucune configuration n'est nécessaire : ouvrez le même fichier dans deux navigateurs (ou deux
|
||||
fenêtres) pour voir la collaboration en action.
|
||||
|
||||
---
|
||||
|
||||
## 🔌 API
|
||||
|
||||
> 📖 Guide complet : [API REST](docs/GUIDES/API_REST.md) · [Serveur MCP](docs/GUIDES/MCP.md)
|
||||
|
||||
ObsiGate expose une API REST complète :
|
||||
|
||||
| Endpoint | Description | Méthode | Auth |
|
||||
@@ -570,13 +646,14 @@ ObsiGate expose une API REST complète :
|
||||
| `/api/file/{vault}/download?path=` | Téléchargement d'un fichier | GET | Oui |
|
||||
| `/api/file/{vault}/save?path=` | Sauvegarder un fichier | PUT | Oui |
|
||||
| `/api/file/{vault}?path=` | Supprimer un fichier | DELETE | Oui |
|
||||
| `/api/search/advanced` | Recherche avancée TF-IDF | GET | Oui |
|
||||
| `/api/search/advanced` | Recherche avancée TF-IDF (+ `semantic=true` pour l'hybride) | GET | Oui |
|
||||
| `/api/suggest` / `/api/tags/suggest` | Autocomplétion | GET | Oui |
|
||||
| `/api/tags?vault=` | Tags uniques avec compteurs | GET | Oui |
|
||||
| `/api/index/reload` | Force un re-scan des vaults | GET | Admin |
|
||||
| `/api/events` | Flux SSE temps réel | GET | Oui |
|
||||
| `/api/vaults/add` / `/api/vaults/{name}` | Gestion dynamique des vaults | POST/DELETE | Admin |
|
||||
| `/api/image/{vault}?path=` | Servir une image | GET | Oui |
|
||||
| `/api/media/{vault}/thumb?path=&size=` | Miniature WebP (cache disque) | GET | Oui |
|
||||
| `/api/config` | Lire / écrire la configuration | GET/POST | Oui/Admin |
|
||||
| `/api/diagnostics` | Statistiques index et mémoire | GET | Admin |
|
||||
|
||||
@@ -597,6 +674,8 @@ curl "http://localhost:2020/api/file/Recettes?path=pizza.md"
|
||||
|
||||
## 🔍 Recherche avancée
|
||||
|
||||
> 📖 Guide complet : [Recherche, PDF, Excel & Excalidraw](docs/GUIDES/RECHERCHE_PDF_EXCALIDRAW.md)
|
||||
|
||||
### Syntaxe de requête
|
||||
|
||||
| Opérateur | Description | Exemple |
|
||||
@@ -611,6 +690,23 @@ curl "http://localhost:2020/api/file/Recettes?path=pizza.md"
|
||||
|
||||
Les opérateurs sont combinables : `tag:linux vault:IT ext:md serveur web`.
|
||||
|
||||
### Support PDF
|
||||
|
||||
Les fichiers PDF de vos vaults s'affichent en ligne dans le navigateur via le visualiseur PDF natif (iframe + `<embed>`).
|
||||
Le visualiseur streame le fichier via HTTP Range (206 Partial Content) — les gros PDF se chargent progressivement.
|
||||
Le texte est extrait à l'indexation (pypdf / pymupdf) — le contenu PDF est donc recherchable via la recherche full-text.
|
||||
Filtrez avec `ext:pdf` pour restreindre les résultats aux PDF.
|
||||
Les métadonnées (pages, titre, auteur) sont disponibles via `GET /api/file/{vault}/pdf/info` sans transférer le document.
|
||||
|
||||
**Limitations :** pas d'OCR (les PDF scannés ne sont pas recherchables), pas d'annotation, pas d'édition du PDF lui-même.
|
||||
|
||||
### Diagrammes Excalidraw
|
||||
|
||||
Les fichiers `.excalidraw` et `.excalidraw.md` s'ouvrent dans un éditeur visuel Excalidraw complet, intégré dans une iframe sandboxée — dessinez, modifiez et sauvegardez sans quitter ObsiGate.
|
||||
Les modifications sont sauvegardées automatiquement (2 s) ou avec `Ctrl+S` ; l'éditeur suit le thème clair/sombre.
|
||||
Le texte des éléments du diagramme est extrait à l'indexation : il est donc recherchable via la recherche full-text (`ext:excalidraw`).
|
||||
Les fichiers créés avec le **plugin Obsidian Excalidraw** (y compris le format `.excalidraw.md` compressé) sont compatibles.
|
||||
|
||||
### Raccourcis clavier
|
||||
|
||||
| Raccourci | Action |
|
||||
@@ -627,6 +723,11 @@ Les opérateurs sont combinables : `tag:linux vault:IT ext:md serveur web`.
|
||||
- **Boost titre** : correspondances dans le titre ×3
|
||||
- **Normalisation des accents** : `resume` trouve `résumé`
|
||||
- **Snippets surlignés** (`<mark>`), **facettes** (compteurs par vault/tag), **pagination** (50/page), **tri** pertinence/date, **chips** de filtres, **historique** (50 recherches)
|
||||
- **Recherche sémantique** (optionnelle) : le toggle `~` (ou `Alt+S`) fusionne le classement
|
||||
TF-IDF avec un classement par embeddings (RRF). Fonctionne sans dépendance avec un provider de
|
||||
hachage ; installez `backend/requirements-semantic.txt` et/ou renseignez `OBSIGATE_EMBEDDING_*`
|
||||
pour de vrais embeddings `all-MiniLM-L6-v2`. Voir
|
||||
[docs/features/semantic-search.md](docs/features/semantic-search.md).
|
||||
|
||||
---
|
||||
|
||||
@@ -696,11 +797,13 @@ Configurables via l'interface (Settings) ou l'API `/api/config`.
|
||||
|
||||
## 🛡️ Sécurité
|
||||
|
||||
> 📖 Guide complet : [Authentification & sécurité](docs/GUIDES/AUTHENTIFICATION_SECURITE.md)
|
||||
|
||||
- **Path traversal** : tous les endpoints fichier valident que le chemin résolu reste dans la vault
|
||||
- **Rate limiting** : 10 tentatives de login max par IP sur 15 minutes + lockout par compte (5 tentatives)
|
||||
- **Audit log** : écritures/suppressions/config journalisées dans `data/audit.log` (JSON lines, rotation 10 MB)
|
||||
- **Backup automatique** : chaque modification/suppression sauvegardée dans `.obsigate-backup/` avec timestamp
|
||||
- **Secret redaction** : masquage automatique des JWT, clés API, tokens dans les aperçus
|
||||
- **Secret redaction** : masquage automatique des JWT, mots de passe, clés API (OpenAI, GitHub, Google, AWS, Slack, Stripe…), tokens dans les aperçus — cliquez sur un masque pour copier la valeur
|
||||
- **Utilisateur non-root** : conteneur Docker sous `obsigate` (UID 1000)
|
||||
- **Volumes read-only** : vaults montées en `:ro` par défaut
|
||||
- **Secrets dans `.env`** : jamais dans `docker-compose.yml`
|
||||
@@ -771,7 +874,7 @@ Configurables via l'interface (Settings) ou l'API `/api/config`.
|
||||
| Validation des imports frontend | `node tests/frontend/validate-imports.mjs` | `lint` |
|
||||
| Tests unitaires frontend | `node tests/frontend/unit.test.mjs` | `lint` |
|
||||
| Tests backend | `pytest tests/ -q` | `test` |
|
||||
| **E2E Playwright** | `npm run test:e2e` (~5 min) | `e2e` |
|
||||
| **E2E Playwright** | `npm run test:e2e` (~10 min) | `e2e` |
|
||||
|
||||
#### Tests E2E locaux (`npm run test:e2e`)
|
||||
|
||||
@@ -794,6 +897,15 @@ bash scripts/run-e2e-local.sh --headed # navigateur visible
|
||||
bash scripts/run-e2e-local.sh -g "reset panes" # filtre sur un test
|
||||
```
|
||||
|
||||
Sous Windows, si `bash` n'est pas exploitable (WSL indisponible, git-bash
|
||||
bloqué par une politique de contrôle d'application), utiliser le lanceur
|
||||
PowerShell équivalent :
|
||||
|
||||
```powershell
|
||||
npm run test:e2e:ps
|
||||
pwsh -File scripts/run-e2e-local.ps1 -PlaywrightArgs @('-g','reset panes')
|
||||
```
|
||||
|
||||
La suite doit se terminer sur **tous les tests passant** (60 actuellement),
|
||||
sans échec ni dépendance aux retries. En cas d'échec : corriger et relancer
|
||||
localement jusqu'à 100 %, puis seulement commiter.
|
||||
@@ -845,7 +957,8 @@ ObsiGate/
|
||||
|
||||
### Contribuer
|
||||
|
||||
Voir [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md) pour les détails.
|
||||
Voir [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md) pour les standards de code et
|
||||
[docs/DELIVERY_WORKFLOW.md](./docs/DELIVERY_WORKFLOW.md) pour la méthode de livraison obligatoire.
|
||||
|
||||
---
|
||||
|
||||
@@ -864,8 +977,8 @@ Ce projet est sous licence **MIT** — voir le fichier [LICENSE](LICENSE) pour l
|
||||
|
||||
## 📝 Changelog
|
||||
|
||||
Consultez le [CHANGELOG.md](./CHANGELOG.md) pour l'historique complet de toutes les versions (v1.0.0 → v2.0.0-dev).
|
||||
Consultez le [CHANGELOG.md](./CHANGELOG.md) pour l'historique complet de toutes les versions (v1.0.0 → v2.86.1).
|
||||
|
||||
---
|
||||
|
||||
*Projet : ObsiGate | Version : 2.0.0-dev | Dernière mise à jour : Juin 2026*
|
||||
*Projet : ObsiGate | Version : 2.86.1 | Dernière mise à jour : Septembre 2026*
|
||||
|
||||
@@ -2,62 +2,89 @@
|
||||
|
||||
**Ultra-light web gateway for your Obsidian vaults** — Access, browse, and search all your Obsidian notes from any device via a modern, responsive web interface.
|
||||
|
||||
[]()
|
||||
[]()
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://www.docker.com/)
|
||||
[](https://www.python.org/)
|
||||
[](https://git.dracodev.net/Projets/ObsiGate/actions)
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ [🔍 Search...] [☀/🌙 Theme] ObsiGate │
|
||||
├──────────────┬──────────────────────────────────────────┤
|
||||
│ SIDEBAR │ CONTENT AREA │
|
||||
│ ▼ Recipes │ 📄 File Title │
|
||||
│ 📁 Soups │ Tags: #recipe #quick │
|
||||
│ 📄 Pizza │ [Rendered Markdown Content] │
|
||||
│ ▼ IT │ │
|
||||
│ 📁 Docker │ │
|
||||
│ Tags Cloud │ │
|
||||
└──────────────┴──────────────────────────────────────────┘
|
||||
```
|
||||

|
||||
|
||||
> ObsiGate web interface: multi-vault sidebar, global search, dashboard stats and shortcuts.
|
||||
|
||||
---
|
||||
|
||||
## 📚 Guides
|
||||
|
||||
Step-by-step **user guides** live in [`docs/GUIDES/`](docs/GUIDES/):
|
||||
|
||||
| Guide | What it covers |
|
||||
|---|---|
|
||||
| 🚀 [Getting Started](docs/GUIDES/PRISE_EN_MAIN.md) | First run, interface, navigation, vaults, shortcuts |
|
||||
| 🔍 [Search, PDF, Excel & Excalidraw](docs/GUIDES/RECHERCHE_PDF_EXCALIDRAW.md) | Query syntax, semantic search, PDF/Excel viewers, diagrams |
|
||||
| 🤖 [AI Assistant & Forge](docs/GUIDES/ASSISTANT_IA_FORGE.md) | Providers, AI editor, BooksLM, Forge, `@` / `/` commands |
|
||||
| 📝 [Editing & Collaboration](docs/GUIDES/COLLABORATION.md) | Simultaneous editing, remote cursors, persistence |
|
||||
| 📱 [PWA & Offline](docs/GUIDES/PWA_HORS_LIGNE.md) | Install as an app, offline cache, sync queue, push |
|
||||
| 🔌 [REST API](docs/GUIDES/API_REST.md) | Authentication, API keys, endpoints, `curl` examples, SSE |
|
||||
| 🧩 [MCP Server](docs/GUIDES/MCP.md) | Connect Claude Desktop, Cursor, Cline… to your vaults |
|
||||
| 🔒 [Auth & Security](docs/GUIDES/AUTHENTIFICATION_SECURITE.md) | Users, MFA, per-vault permissions, hardening |
|
||||
| 🐳 [Docker Deployment](docs/GUIDES/DEPLOIEMENT_DOCKER.md) | `docker-compose`, volumes, reverse proxy, updates |
|
||||
| 🖥️ [Desktop (Tauri)](docs/GUIDES/DESKTOP.md) | Install, first run, build from source, troubleshooting |
|
||||
|
||||
> All guides are currently written in **French**. See the full index:
|
||||
> [`docs/GUIDES/README.md`](docs/GUIDES/README.md).
|
||||
|
||||
---
|
||||
|
||||
## 📋 Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Architecture](#architecture)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Quick Installation](#quick-installation)
|
||||
- [Detailed Configuration](#detailed-configuration)
|
||||
- [Environment Variables](#environment-variables)
|
||||
- [🔒 Authentication](#authentication)
|
||||
- [Adding a New Vault](#adding-a-new-vault)
|
||||
- [Build & Deployment with build.sh](#build--deployment-with-buildsh)
|
||||
- [Desktop (Tauri) — Native Application](#desktop-tauri--native-application)
|
||||
- [Usage](#usage)
|
||||
- [API](#api)
|
||||
- [Performance](#performance)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Tech Stack](#tech-stack)
|
||||
- [Changelog](#changelog)
|
||||
- ✨ [Features](#features)
|
||||
- 📚 [Guides](#guides)
|
||||
- 🚀 [Prerequisites](#prerequisites)
|
||||
- ⚡ [Quick Installation](#quick-installation)
|
||||
- ⚙️ [Detailed Configuration](#detailed-configuration)
|
||||
- 🌍 [Environment Variables](#environment-variables)
|
||||
- 🔒 [Authentication](#authentication)
|
||||
- ➕ [Adding a New Vault](#adding-a-new-vault)
|
||||
- 🔨 [Build & Deployment with build.sh](#build--deployment-with-buildsh)
|
||||
- 🖼️ [Obsidian Image Rendering](#obsidian-image-rendering)
|
||||
- 🖥️ [Desktop (Tauri) — Native Application](#desktop-tauri--native-application)
|
||||
- 📖 [Usage](#usage)
|
||||
- 👥 [Real-time Collaboration](#real-time-collaboration)
|
||||
- 🔌 [API](#api)
|
||||
- 🔍 [Advanced Search](#advanced-search)
|
||||
- 🛡️ [Security](#security)
|
||||
- ⚡ [Performance](#performance)
|
||||
- 🔧 [Troubleshooting](#troubleshooting)
|
||||
- 🏗️ [Tech Stack](#tech-stack)
|
||||
- 🏠 [Architecture](#architecture)
|
||||
- 📝 [Development](#development)
|
||||
- 📄 [License](#license)
|
||||
- 🤝 [Support](#support)
|
||||
- 📝 [Changelog](#changelog)
|
||||
|
||||
---
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- **🤖 Integrated AI Editor** — CodeMirror 6 editor with AI toolbar: improve, correct, translate, generate, custom rewrite, toolbox (list, table, frontmatter, canvas) — multi-provider DeepSeek/OpenRouter/Gemini
|
||||
- **🧩 MCP Server & AI Agent** — Built-in Model Context Protocol server (`/mcp`) and tool-calling assistant: read, search and edit your vaults from Claude Desktop, Cursor… with two-step confirmations, per-vault permissions, rate limiting and secret redaction ([guide](docs/GUIDES/MCP.md))
|
||||
- **👥 Real-time Collaboration** — Simultaneous editing of the same document (Yjs/CRDT): colored remote cursors, presence indicator, conflict-free merge, automatic reconnection and server-side persistence ([details](docs/features/collaboration.md))
|
||||
- **📖 Built-in User Guide** — Complete FR/EN help from the Options menu: interface, navigation, search, files, AI, security, API & integrations (OpenAPI, MCP), offline, collaboration, desktop, plus an **Architecture** section with a Mermaid diagram; downloadable as **Markdown** and **PDF** in the current language ([details](docs/features/guide-coverage-105.md))
|
||||
- **📱 Native Mobile Editor** — Touch-optimised editing: floating Markdown toolbar (bold/italic/code/list/link), persistent Paste button (iOS workaround), pinch-zoom font & adjustable height, swipe shortcuts (backlinks / table of contents) and a full-screen reading mode with page navigation ([details](docs/features/mobile-editor.md))
|
||||
- **🗺️ Interactive Graph View** — Canvas force-directed with Barnes-Hut O(n log n), filters (tag, type), depth, focus mode, navigation history ←→↑, export PNG, preview on hover (Ctrl+click)
|
||||
- **🗂️ Multi-vault** : View multiple Obsidian vaults simultaneously
|
||||
- **🌳 Tree Navigation** : Browse your folders and files in the sidebar
|
||||
- **🔍 Advanced Search** : TF-IDF search engine with French stemming, accent normalization, highlighted snippets, facets, pagination, and sorting
|
||||
- **🌳 Tree Navigation** : Browse your folders and files in the sidebar; clicking a folder in the tree opens a dedicated **navigation tab** — path, recents, clickable subfolders, Vaults · Tags · Extensions facets, Pertinence/Date sorting and save-as-search — coexisting with your open files
|
||||
- **🔍 Advanced Search** : TF-IDF search engine with French stemming, accent normalization, highlighted snippets, facets, pagination, and sorting — plus an optional **semantic search** (embeddings via `all-MiniLM-L6-v2`, hybrid TF-IDF + RRF fusion) toggled with `~` ([details](docs/features/semantic-search.md))
|
||||
- **💡 Smart Autocomplete** : Suggestions for files, tags, and history with keyboard navigation
|
||||
- **🧩 Query Syntax** : Operators `tag:`, `#`, `vault:`, `title:`, `path:`, `ext:` with visual chips
|
||||
- **📜 Search History** : Persisted in localStorage (max 50 entries, LIFO, deduplicated)
|
||||
- **🏷️ Tag Cloud** : Filtering by tags extracted from YAML frontmatters
|
||||
- **🔗 Wikilinks** : `[[internal links]]` from Obsidian are clickable
|
||||
- **🖼️ Obsidian Images** : Full support for all Obsidian image syntaxes with intelligent resolution
|
||||
- **🎬 Audio & video** : Built-in HTML5 players (`.mp3 .wav .flac .mp4 .webm`…) with HTTP Range streaming (play, seek, fullscreen) and **persistent playback** (floating mini-player / mini video window, return to media or stop anytime, lock-screen controls via Media Session), falling back to download when the format is not playable in the browser
|
||||
- **🎨 Excalidraw Diagrams** : Native viewer/editor for `.excalidraw` and `.excalidraw.md` files (sandboxed iframe, autosave, dark/light theme, diagram text indexed for search)
|
||||
- **📊 Excel Spreadsheets** : `.xlsx` and `.xlsm` files open in a dedicated viewer — one table per sheet with tabs, A1 headers and inline cell editing (`PUT /api/file/{vault}/xlsx/save`, automatic backup, atomic write), plus download of the original file. The viewer renders fonts, colors, merged cells and frozen panes, offers keyboard navigation and shortcuts (`Ctrl+S`, `Delete`, `F2`, `Ctrl+Home/End`, `PgUp/PgDn`, `Ctrl+arrows`), a formula bar with function suggestions, an editable Name Box ("go to" `A1:B3`), a range clipboard (copy/cut/paste a block, from or to Excel), a **Format** menu (bold/italic/underline, alignments, font & fill colours, number formats, merges, frozen panes, column width/row height — `PUT /api/file/{vault}/xlsx/style`), sort/filter/find across every sheet, CSV/Markdown/HTML export and printing (selection or sheet), sheet & row/column structure editing and a workbook dashboard (named ranges, charts/pivot detection, per-sheet stats); `.csv` is edited in the same grid (RFC 4180) while `.xls` and `.ods` open read-only. Workbooks holding elements ObsiGate cannot preserve (cached values, slicers, form controls, signature…) show a **warning** and ask for confirmation before saving; a value starting with `=` or `@` is stored as text unless the `f(x)` toggle is enabled, and concurrent writes from another process are detected (`If-Match` → "Retry"). The AI assistant can list sheets, dump a bounded table to its context, search the workbook, analyze a range, update cells and append rows — on `.xlsx`, `.xlsm` and `.csv`. On mobile (≤ 768 px), the menu bar and ribbon are **collapsed by default** — a single ☰ button expands them — so the grid gets the full screen height
|
||||
- **🎨 Syntax Highlight** : Syntax highlighting for code blocks
|
||||
- **🌓 Light/Dark Theme** : Toggle persisted in localStorage
|
||||
- **📡 Real-time Sync** : Automatic file monitoring via watchdog with incremental index updates
|
||||
@@ -211,6 +238,7 @@ Vaults are configured using pairs of `VAULT_N_NAME` / `VAULT_N_PATH` variables (
|
||||
| `VAULT_1_SCAN_ATTACHMENTS` | Enable image scanning on startup (optional, default: true) | `true` |
|
||||
| `VAULT_2_NAME` | Display name of the vault | `IT` |
|
||||
| `VAULT_2_PATH` | Path inside the container | `/vaults/Obsidian_IT` |
|
||||
| `OBSIGATE_HOME_ROOT` | Root folder for per-user home directories (one `home-<user>` vault per account). Unset = feature disabled. | `/vaults/Home` |
|
||||
|
||||
**Naming rules:**
|
||||
- Use only letters, numbers, and hyphens
|
||||
@@ -309,7 +337,24 @@ When an **admin** account is logged in, a 🛡️ icon appears in the header. Cl
|
||||
| `OBSIGATE_ACCESS_TOKEN_TTL` | JWT token lifetime (seconds) | `3600` |
|
||||
| `OBSIGATE_REFRESH_TOKEN_TTL` | Refresh token lifetime (seconds) | `2592000` |
|
||||
| `OBSIGATE_LOGIN_MAX_ATTEMPTS` | Max login attempts per IP | `10` |
|
||||
| `OBSIGATE_ACCOUNT_MAX_ATTEMPTS` | Max login attempts per account | `10` |
|
||||
| `OBSIGATE_LOGIN_WINDOW_SECONDS` | Rate limiting window (seconds) | `900` |
|
||||
| `OBSIGATE_TRUST_PROXY` | Trust `X-Forwarded-For` for the client IP (reverse proxy) | `false` |
|
||||
| `OBSIGATE_WEBHOOK_ALLOW_HTTP` | Allow non-HTTPS webhook targets | `false` |
|
||||
| `OBSIGATE_WEBHOOK_ALLOW_PRIVATE` | Allow webhooks to private/loopback addresses | `false` |
|
||||
| `OBSIGATE_PDF_MAX_SIZE_MB` | Max PDF size for text extraction | `50` |
|
||||
| `OBSIGATE_MEDIA_MAX_INLINE_MB` | Max size for inline audio/video playback (above: download) | `500` |
|
||||
| `OBSIGATE_PDF_EXTRACT_TIMEOUT` | PDF extraction timeout (seconds) | `30` |
|
||||
| `OBSIGATE_TAVILY_API_KEY` / `OBSIGATE_BRAVE_API_KEY` / `OBSIGATE_SERPAPI_API_KEY` / `OBSIGATE_EXA_API_KEY` | Keyed web-search providers (tried before SearXNG) | — |
|
||||
| `OBSIGATE_WEB_PROVIDERS` | Search provider order (e.g. `brave,searxng`) | — |
|
||||
| `OBSIGATE_WEB_RETRY` | Web tools network retries (house-made backoff) | `1` |
|
||||
| `OBSIGATE_WEB_CACHE_TTL` | SQLite cache TTL for web results (seconds, `0` = off) | `900` |
|
||||
| `OBSIGATE_GITEA_URL` / `OBSIGATE_GITEA_TOKEN` | Gitea connected source (`git_list_repos`…) | — |
|
||||
| `OBSIGATE_GITHUB_TOKEN` | GitHub token (`git_list_repos`…) | — |
|
||||
|
||||
> These keys can also be entered **from the UI** (menu → Configurations →
|
||||
> "Connected sources & search"): the stored value goes to `data/api_keys.json`
|
||||
> and takes precedence over the environment variable.
|
||||
|
||||
>All these variables are documented in `.env.example`.
|
||||
|
||||
@@ -475,6 +520,17 @@ ObsiGate uses 7 resolution strategies in order of priority:
|
||||
6. **Startup index (closest match)** : If multiple files have the same name
|
||||
7. **Fallback** : Display a styled placeholder `[image not found: filename.ext]`
|
||||
|
||||
### Viewer & file tree
|
||||
|
||||
Images are first-class vault files: they appear in the tree, are indexed (name +
|
||||
metadata, **never the bytes**) and open in a **dedicated viewer** — wheel zoom
|
||||
0.1×–8×, drag pan, double-click to reset, ←/→ navigation between images in the
|
||||
same folder (WebP thumbnail filmstrip), metadata panel, full-screen lightbox,
|
||||
open original and download. The `ext:png`/`ext:jpg` search filter is available.
|
||||
Decodable formats: PNG, JPEG, GIF, WebP, BMP, ICO, SVG (SVG served with a
|
||||
`sandbox` CSP). **HEIC/HEIF** (iPhone) is not decodable by browsers and is not
|
||||
supported.
|
||||
|
||||
### Configuration
|
||||
|
||||
To optimize resolution, configure the attachments folder for each vault:
|
||||
@@ -499,6 +555,8 @@ curl -X POST http://localhost:2020/api/attachments/rescan/MyVault
|
||||
|
||||
## 🖥️ Desktop (Tauri) — Native Application
|
||||
|
||||
> 📖 Full guide: [Desktop (Tauri)](docs/GUIDES/DESKTOP.md)
|
||||
|
||||
ObsiGate Desktop is a native application built with [Tauri](https://tauri.app/) (Rust + system webview). It embeds the Python backend and frontend in a standalone executable — zero Docker, zero command line.
|
||||
|
||||
> 🚧 **Version 2.0.0 — binaries are being stabilized.** For now, building from source is recommended.
|
||||
@@ -664,8 +722,28 @@ Lifecycle: Tauri spawns the Python backend → health check → opens the webvie
|
||||
|
||||
---
|
||||
|
||||
## 👥 Real-time Collaboration
|
||||
|
||||
> 📖 Full guide: [Editing & Collaboration](docs/GUIDES/COLLABORATION.md)
|
||||
|
||||
Multiple users can edit the same markdown document simultaneously (Google Docs style):
|
||||
|
||||
- **Conflict-free merge** via Yjs (CRDT): two people can type in the same place, no change is lost.
|
||||
- **Colored remote cursors** and visible selections in CodeMirror, labelled with each user's name.
|
||||
- **Presence indicator** in the editor header (avatars + connection status).
|
||||
- **Automatic reconnection** (exponential backoff): state is merged on return.
|
||||
- **Server-side persistence**: the document is written to disk 2 s after the last change.
|
||||
- **Transport**: WebSocket `ws(s)://<host>/ws/collab/{vault}/{path}`, authenticated via the
|
||||
`access_token` cookie (or `?token=`) and subject to per-vault access control.
|
||||
|
||||
No configuration is required: open the same file in two browsers (or two windows) to see it live.
|
||||
|
||||
---
|
||||
|
||||
## 🔌 API
|
||||
|
||||
> 📖 Full guide: [REST API](docs/GUIDES/API_REST.md) · [MCP Server](docs/GUIDES/MCP.md)
|
||||
|
||||
ObsiGate exposes a complete REST API :
|
||||
|
||||
| Endpoint | Description | Method | Auth |
|
||||
@@ -686,13 +764,14 @@ ObsiGate exposes a complete REST API :
|
||||
| `/api/file/{vault}/download?path=` | Download a file | GET | Yes |
|
||||
| `/api/file/{vault}/save?path=` | Save a file | PUT | Yes |
|
||||
| `/api/file/{vault}?path=` | Delete a file | DELETE | Yes |
|
||||
| `/api/search/advanced` | Advanced TF-IDF search | GET | Yes |
|
||||
| `/api/search/advanced` | Advanced TF-IDF search (+ `semantic=true` for hybrid) | GET | Yes |
|
||||
| `/api/suggest` / `/api/tags/suggest` | Autocomplete | GET | Yes |
|
||||
| `/api/tags?vault=` | Unique tags with counters | GET | Yes |
|
||||
| `/api/index/reload` | Force a rescan of vaults | GET | Admin |
|
||||
| `/api/events` | Real-time SSE stream | GET | Yes |
|
||||
| `/api/vaults/add` / `/api/vaults/{name}` | Dynamic vault management | POST/DELETE | Admin |
|
||||
| `/api/image/{vault}?path=` | Serve an image | GET | Yes |
|
||||
| `/api/media/{vault}/thumb?path=&size=` | WebP thumbnail (disk cache) | GET | Yes |
|
||||
| `/api/config` | Read / write configuration | GET/POST | Yes/Admin |
|
||||
| `/api/diagnostics` | Index and memory statistics | GET | Admin |
|
||||
|
||||
@@ -726,6 +805,8 @@ curl "http://localhost:2020/api/file/Recipes?path=pizza.md"
|
||||
|
||||
## 🔍 Advanced Search
|
||||
|
||||
> 📖 Full guide: [Search, PDF, Excel & Excalidraw](docs/GUIDES/RECHERCHE_PDF_EXCALIDRAW.md)
|
||||
|
||||
### Query Syntax
|
||||
|
||||
| Operator | Description | Example |
|
||||
@@ -738,7 +819,24 @@ curl "http://localhost:2020/api/file/Recipes?path=pizza.md"
|
||||
| `ext:<type>` | Filter by file type | `ext:md kubernetes` |
|
||||
| `"exact phrase"` | Phrase search | `tag:"multiple words"` |
|
||||
|
||||
Extension filter examples: `ext:sh` for bash scripts, `ext:py` for Python scripts, `ext:md` for Markdown files.
|
||||
Extension filter examples: `ext:sh` for bash scripts, `ext:py` for Python scripts, `ext:md` for Markdown files, `ext:pdf` for PDF documents, `ext:excalidraw` for Excalidraw diagrams (text-extracted content is indexed).
|
||||
|
||||
### PDF support
|
||||
|
||||
PDF files in your vaults are rendered inline in the browser via the native PDF viewer (iframe + `<embed>`).
|
||||
The viewer streams the file over HTTP Range requests (206 Partial Content), so large PDFs load progressively.
|
||||
Text is extracted on indexing (pypdf / pymupdf) so PDF content is searchable via the full-text search.
|
||||
Filter with `ext:pdf` to restrict results to PDFs.
|
||||
PDF metadata (pages, title, author) is available via `GET /api/file/{vault}/pdf/info` without transferring the document.
|
||||
|
||||
**Limitations:** no OCR (scanned PDFs aren't searchable), no annotation, no editing of the PDF itself.
|
||||
|
||||
### Excalidraw diagrams
|
||||
|
||||
`.excalidraw` and `.excalidraw.md` files open in a full visual Excalidraw editor embedded in a sandboxed iframe — draw, edit and save without leaving ObsiGate.
|
||||
Changes are saved automatically (2s debounce) or with `Ctrl+S`; the editor follows the light/dark theme.
|
||||
Text labels inside the diagram elements are extracted on indexing, so diagram content is searchable via the full-text search (`ext:excalidraw`).
|
||||
Files created with the **Obsidian Excalidraw plugin** (including the compressed `.excalidraw.md` format) are compatible.
|
||||
|
||||
Operators are combinable: `tag:linux vault:IT ext:md server web` searches for "server web" in Markdown files of the IT vault with the linux tag.
|
||||
|
||||
@@ -763,6 +861,10 @@ Operators are combinable: `tag:linux vault:IT ext:md server web` searches for "s
|
||||
- **Sorting** : By relevance (TF-IDF) or modification date
|
||||
- **Visual chips** : Active filters are shown as removable colored chips
|
||||
- **History** : Last 50 searches are stored in localStorage
|
||||
- **Semantic search** (optional) : Toggle `~` (or `Alt+S`) fuses the TF-IDF ranking with an
|
||||
embedding-based ranking (RRF). Works out of the box with a dependency-free hashing embedder;
|
||||
install `backend/requirements-semantic.txt` and/or set `OBSIGATE_EMBEDDING_*` for real
|
||||
`all-MiniLM-L6-v2` embeddings. See [docs/features/semantic-search.md](docs/features/semantic-search.md).
|
||||
|
||||
---
|
||||
|
||||
@@ -857,11 +959,13 @@ These parameters are configurable via the interface (Settings) or the `/api/conf
|
||||
|
||||
## 🛡️ Security
|
||||
|
||||
> 📖 Full guide: [Auth & Security](docs/GUIDES/AUTHENTIFICATION_SECURITE.md)
|
||||
|
||||
- **Path traversal** : All file endpoints validate that the resolved path stays within the vault
|
||||
- **Rate limiting** : 10 login attempts max per IP over 15 minutes + per-account lockout (5 attempts)
|
||||
- **Audit log** : All writes, deletions, and config changes are logged in `data/audit.log` (JSON lines, 10 MB rotation)
|
||||
- **Automatic backup** : Every file modification or deletion is saved in `.obsigate-backup/` with timestamp
|
||||
- **Secret redaction** : Automatic masking of JWTs, API keys, tokens, and connection strings in previews
|
||||
- **Secret redaction** : Automatic masking of JWTs, passwords, API keys (OpenAI, GitHub, Google, AWS, Slack, Stripe…), tokens and connection strings in previews — click a mask to copy the value
|
||||
- **Non-root user** : The Docker container runs under user `obsigate` (UID 1000)
|
||||
- **Read-only volumes** : Vaults are mounted as `:ro` by default in docker-compose
|
||||
- **Secrets in `.env`** : Passwords and tokens are never in `docker-compose.yml`
|
||||
@@ -940,7 +1044,7 @@ These parameters are configurable via the interface (Settings) or the `/api/conf
|
||||
| Frontend import validation | `node tests/frontend/validate-imports.mjs` | `lint` |
|
||||
| Frontend unit tests | `node tests/frontend/unit.test.mjs` | `lint` |
|
||||
| Backend tests | `pytest tests/ -q` | `test` |
|
||||
| **E2E Playwright** | `npm run test:e2e` (~5 min) | `e2e` |
|
||||
| **E2E Playwright** | `npm run test:e2e` (~10 min) | `e2e` |
|
||||
|
||||
#### Local E2E Tests (`npm run test:e2e`)
|
||||
|
||||
@@ -963,6 +1067,14 @@ bash scripts/run-e2e-local.sh --headed # visible browser
|
||||
bash scripts/run-e2e-local.sh -g "reset panes" # filter on a test
|
||||
```
|
||||
|
||||
On Windows, when `bash` is unusable (WSL unavailable, git-bash blocked by an
|
||||
Application Control policy), use the equivalent PowerShell launcher:
|
||||
|
||||
```powershell
|
||||
npm run test:e2e:ps
|
||||
pwsh -File scripts/run-e2e-local.ps1 -PlaywrightArgs @('-g','reset panes')
|
||||
```
|
||||
|
||||
The suite must end with **all tests passing** (60 currently), with no failure
|
||||
or reliance on retries. In case of failure: fix and re-run locally until 100 %,
|
||||
then only commit.
|
||||
@@ -1012,12 +1124,15 @@ ObsiGate/
|
||||
├── Dockerfile # Multi-stage, healthcheck, non-root
|
||||
├── docker-compose.yml # Deployment with healthcheck and auth env vars
|
||||
├── build.sh # Automated build & deployment (docker compose build + up)
|
||||
└── CONTRIBUTING.md # Contribution guide
|
||||
└── docs/
|
||||
├── GUIDES/ # User guides (getting started, API, MCP, desktop…)
|
||||
└── CONTRIBUTING.md # Contribution guide
|
||||
```
|
||||
|
||||
### Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
||||
See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for code standards and
|
||||
[docs/DELIVERY_WORKFLOW.md](docs/DELIVERY_WORKFLOW.md) for the mandatory delivery process.
|
||||
|
||||
---
|
||||
|
||||
@@ -1037,8 +1152,8 @@ This project is licensed under the **MIT License** - see the [LICENSE](LICENSE)
|
||||
|
||||
## 📝 Changelog
|
||||
|
||||
See [CHANGELOG.md](./CHANGELOG.md) for the complete version history (v1.0.0 → v1.7.0).
|
||||
See [CHANGELOG.md](./CHANGELOG.md) for the complete version history (v1.0.0 → v2.86.1).
|
||||
|
||||
---
|
||||
|
||||
*Project: ObsiGate | Version: 1.7.0 | Last updated: May 2026*
|
||||
*Project: ObsiGate | Version: 2.86.1 | Last updated: September 2026*
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
# backend/admin.py
|
||||
# Admin Dashboard endpoints — system stats, audit logs, backup stats,
|
||||
# and Server-Sent Events stream for real-time widgets.
|
||||
#
|
||||
# All endpoints protected with require_admin.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from backend.audit import AUDIT_LOG_FILE, get_recent_entries
|
||||
from backend.auth.middleware import require_admin
|
||||
from backend.indexer import get_vault_data, index
|
||||
|
||||
logger = logging.getLogger("obsigate.admin")
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
# Server start time — set at module import. Approximates uptime even
|
||||
# if used before lifespan startup fully runs.
|
||||
_SERVER_START_TIME = time.time()
|
||||
|
||||
|
||||
def _server_uptime_seconds() -> float:
|
||||
return time.time() - _SERVER_START_TIME
|
||||
|
||||
|
||||
def _format_timestamp_for_stream(payload: dict) -> str:
|
||||
"""Serialize a payload as an SSE-compatible data line."""
|
||||
return json.dumps(payload, default=str, ensure_ascii=False)
|
||||
|
||||
|
||||
def _get_disk_stats() -> tuple[float, float]:
|
||||
"""Return (used_gb, total_gb) for the data volume or cwd fallback."""
|
||||
# Prefer /data mount when present (production), else use cwd
|
||||
target = "/data" if os.path.isdir("/data") else "."
|
||||
try:
|
||||
usage = shutil.disk_usage(target)
|
||||
used_gb = round(usage.used / (1024 ** 3), 2)
|
||||
total_gb = round(usage.total / (1024 ** 3), 2)
|
||||
return used_gb, total_gb
|
||||
except OSError as e:
|
||||
logger.warning(f"disk_usage failed for {target}: {e}")
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def _count_active_sessions() -> int:
|
||||
"""Count currently-revoked-free active sessions.
|
||||
|
||||
Best-effort estimate: we don't keep an in-memory session store, so we
|
||||
fall back to 1 if the server is up. Useful for the dashboard tile.
|
||||
"""
|
||||
return 1
|
||||
|
||||
|
||||
# ── /api/admin/stats ────────────────────────────────────────────────────
|
||||
@router.get("/stats")
|
||||
async def get_admin_stats(_admin=Depends(require_admin)):
|
||||
"""Return real-time system stats for the admin dashboard."""
|
||||
import psutil
|
||||
|
||||
try:
|
||||
cpu_pct = psutil.cpu_percent(interval=None)
|
||||
except Exception:
|
||||
cpu_pct = 0.0
|
||||
|
||||
try:
|
||||
vm = psutil.virtual_memory()
|
||||
mem_used_mb = round(vm.used / (1024 ** 2), 1)
|
||||
mem_total_mb = round(vm.total / (1024 ** 2), 1)
|
||||
except Exception:
|
||||
mem_used_mb, mem_total_mb = 0.0, 0.0
|
||||
|
||||
disk_used_gb, disk_total_gb = _get_disk_stats()
|
||||
uptime_seconds = int(_server_uptime_seconds())
|
||||
active_sessions = _count_active_sessions()
|
||||
|
||||
return {
|
||||
"cpu_pct": cpu_pct,
|
||||
"mem_used_mb": mem_used_mb,
|
||||
"mem_total_mb": mem_total_mb,
|
||||
"disk_used_gb": disk_used_gb,
|
||||
"disk_total_gb": disk_total_gb,
|
||||
"uptime_seconds": uptime_seconds,
|
||||
"active_sessions": active_sessions,
|
||||
}
|
||||
|
||||
|
||||
# ── /api/admin/audit ───────────────────────────────────────────────────
|
||||
@router.get("/audit")
|
||||
async def get_admin_audit(
|
||||
user: str | None = Query(None, description="Filter by username (substring)"),
|
||||
action: str | None = Query(None, description="Filter by exact action"),
|
||||
limit: int = Query(500, ge=1, le=2000, description="Max entries"),
|
||||
offset: int = Query(0, ge=0, description="Skip first N entries"),
|
||||
_admin=Depends(require_admin),
|
||||
):
|
||||
"""Return recent audit log entries with optional filters."""
|
||||
entries = get_recent_entries(limit=offset + limit, action=action)
|
||||
|
||||
if user:
|
||||
needle = user.lower()
|
||||
entries = [e for e in entries if needle in str(e.get("username", "")).lower()]
|
||||
|
||||
if offset:
|
||||
entries = entries[offset:]
|
||||
|
||||
return {
|
||||
"entries": entries,
|
||||
"total": len(entries),
|
||||
"log_file": str(AUDIT_LOG_FILE),
|
||||
}
|
||||
|
||||
|
||||
# ── /api/admin/backup-stats ───────────────────────────────────────────
|
||||
def _scan_backups() -> list[dict]:
|
||||
"""Walk every vault's backup directory and return one row per .bak file."""
|
||||
rows: list[dict] = []
|
||||
for vault_name in list(index.keys()):
|
||||
vd = get_vault_data(vault_name)
|
||||
if not vd:
|
||||
continue
|
||||
vault_root = Path(vd["path"])
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
if not backup_root.is_absolute():
|
||||
backup_root = vault_root / backup_root
|
||||
vault_backup_dir = backup_root / vault_name
|
||||
if not vault_backup_dir.exists():
|
||||
continue
|
||||
try:
|
||||
for fpath in vault_backup_dir.rglob("*.bak"):
|
||||
if not fpath.is_file():
|
||||
continue
|
||||
try:
|
||||
st = fpath.stat()
|
||||
except OSError:
|
||||
continue
|
||||
# Backup filename: {orig}.{timestamp}.bak — split from the right
|
||||
name = fpath.name
|
||||
parts = name.rsplit(".", 2)
|
||||
if len(parts) < 3 or not parts[-2].isdigit():
|
||||
continue
|
||||
try:
|
||||
ts = int(parts[-2])
|
||||
except ValueError:
|
||||
continue
|
||||
rows.append({
|
||||
"vault": vault_name,
|
||||
"filename": name,
|
||||
"timestamp": ts,
|
||||
"size": st.st_size,
|
||||
})
|
||||
except OSError as e:
|
||||
logger.warning(f"Backup scan error in {vault_backup_dir}: {e}")
|
||||
rows.sort(key=lambda r: r["timestamp"], reverse=True)
|
||||
return rows
|
||||
|
||||
|
||||
@router.get("/backup-stats")
|
||||
async def get_admin_backup_stats(_admin=Depends(require_admin)):
|
||||
"""Return backup statistics across all vaults."""
|
||||
rows = _scan_backups()
|
||||
now_ts = int(time.time())
|
||||
total_size_mb = round(sum(r["size"] for r in rows) / (1024 ** 2), 2)
|
||||
|
||||
by_vault: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
bucket = by_vault.setdefault(r["vault"], {"count": 0, "size_mb": 0.0})
|
||||
bucket["count"] += 1
|
||||
bucket["size_mb"] = round(bucket["size_mb"] + r["size"] / (1024 ** 2), 2)
|
||||
|
||||
if rows:
|
||||
newest_ts = rows[0]["timestamp"]
|
||||
oldest_ts = rows[-1]["timestamp"]
|
||||
newest_age_days = round((now_ts - newest_ts) / 86400, 2)
|
||||
oldest_age_days = round((now_ts - oldest_ts) / 86400, 2)
|
||||
else:
|
||||
newest_age_days = 0.0
|
||||
oldest_age_days = 0.0
|
||||
|
||||
return {
|
||||
"total_backups": len(rows),
|
||||
"total_size_mb": total_size_mb,
|
||||
"oldest_age_days": oldest_age_days,
|
||||
"newest_age_days": newest_age_days,
|
||||
"by_vault": by_vault,
|
||||
}
|
||||
|
||||
|
||||
# ── /api/admin/stream — Server-Sent Events ─────────────────────────────
|
||||
async def _stats_event_generator():
|
||||
"""Yield an SSE `stats` event with the current system stats."""
|
||||
try:
|
||||
import psutil
|
||||
# Prime cpu_percent so the first real measurement isn't 0.0
|
||||
psutil.cpu_percent(interval=None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
last_keepalive = time.time()
|
||||
KEEPALIVE_INTERVAL = 15.0
|
||||
STATS_INTERVAL = 5.0
|
||||
|
||||
try:
|
||||
while True:
|
||||
now = time.time()
|
||||
try:
|
||||
data = await asyncio.to_thread(_build_stats_payload)
|
||||
except Exception as e:
|
||||
logger.warning(f"SSE stats build failed: {e}")
|
||||
data = {"error": str(e)}
|
||||
|
||||
yield f"event: stats\ndata: {_format_timestamp_for_stream(data)}\n\n"
|
||||
|
||||
if now - last_keepalive >= KEEPALIVE_INTERVAL:
|
||||
last_keepalive = now
|
||||
yield ": keepalive\n\n"
|
||||
|
||||
await asyncio.sleep(STATS_INTERVAL)
|
||||
except asyncio.CancelledError:
|
||||
logger.info("SSE stats stream cancelled")
|
||||
raise
|
||||
|
||||
|
||||
def _build_stats_payload() -> dict:
|
||||
"""Synchronous stats builder for the SSE generator."""
|
||||
import psutil
|
||||
cpu_pct = psutil.cpu_percent(interval=None)
|
||||
vm = psutil.virtual_memory()
|
||||
disk_used_gb, disk_total_gb = _get_disk_stats()
|
||||
return {
|
||||
"cpu_pct": cpu_pct,
|
||||
"mem_used_mb": round(vm.used / (1024 ** 2), 1),
|
||||
"mem_total_mb": round(vm.total / (1024 ** 2), 1),
|
||||
"disk_used_gb": disk_used_gb,
|
||||
"disk_total_gb": disk_total_gb,
|
||||
"uptime_seconds": int(_server_uptime_seconds()),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/stream")
|
||||
async def stream_admin_stats(_admin=Depends(require_admin)):
|
||||
"""Server-Sent Events stream of system metrics, every 5 seconds."""
|
||||
return StreamingResponse(
|
||||
_stats_event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,455 @@
|
||||
"""In-app agent loop — multi-step tool calling.
|
||||
|
||||
The loop drives an LLM that may request tool calls, executes them through the
|
||||
shared tool layer (``backend.tools``), feeds the results back, and repeats
|
||||
until the model produces a final answer or the iteration budget is exhausted.
|
||||
|
||||
The LLM is injected as an async callable so the loop is fully testable without
|
||||
network access::
|
||||
|
||||
async def fake_llm(messages, tools):
|
||||
return LLMResponse(content="done")
|
||||
|
||||
result = await run_agent(messages, ctx=ctx, llm=fake_llm)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from backend.tools.api import (
|
||||
ToolConfirmationRequired,
|
||||
ToolContext,
|
||||
ToolError,
|
||||
ToolScope,
|
||||
call_tool,
|
||||
get_tool,
|
||||
get_tool_schemas,
|
||||
)
|
||||
from backend.tools.labels import thought_step_label, tool_step_label
|
||||
|
||||
logger = logging.getLogger("obsigate.agent.loop")
|
||||
|
||||
DEFAULT_MAX_ITERATIONS = 10
|
||||
# Cap the size of a tool result fed back to the model (chars).
|
||||
MAX_TOOL_RESULT_CHARS = 100_000
|
||||
# Quota: maximum tool calls executed per agent run (``BOOKSLM_MAX_TOOL_CALLS``).
|
||||
DEFAULT_MAX_TOOL_CALLS = int(os.environ.get("BOOKSLM_MAX_TOOL_CALLS", "25"))
|
||||
|
||||
# Sent as a last user turn when the loop stopped before the model produced an
|
||||
# answer (iteration/quota budget exhausted while it was still calling tools).
|
||||
_FINALIZE_INSTRUCTION = (
|
||||
"N'appelle plus aucun outil. Réponds maintenant directement à l'utilisateur, "
|
||||
"en français, à partir des informations déjà recueillies ci-dessus. "
|
||||
"Structure la réponse en Markdown, cite les liens sources utiles, et si les "
|
||||
"informations sont insuffisantes, dis-le explicitement."
|
||||
)
|
||||
|
||||
# Stopping reasons
|
||||
STOP_DONE = "done"
|
||||
STOP_MAX_ITERATIONS = "max_iterations"
|
||||
STOP_CONFIRMATION_REQUIRED = "confirmation_required"
|
||||
STOP_QUOTA_EXCEEDED = "quota_exceeded"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallRecord:
|
||||
"""Audit-friendly record of one executed tool call."""
|
||||
|
||||
name: str
|
||||
arguments: dict[str, Any]
|
||||
ok: bool
|
||||
result: Any
|
||||
# Human-readable « step » label for the Notion-style UI
|
||||
# ({key, params} — see backend.tools.labels).
|
||||
step: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentResult:
|
||||
"""Outcome of an agent run."""
|
||||
|
||||
content: str = ""
|
||||
messages: list[dict[str, Any]] = field(default_factory=list)
|
||||
tool_calls: list[ToolCallRecord] = field(default_factory=list)
|
||||
# Ordered Notion-style step descriptors ({key, params}); tool steps and
|
||||
# intermediate reasoning notes interleaved by execution order.
|
||||
steps: list[dict[str, Any]] = field(default_factory=list)
|
||||
iterations: int = 0
|
||||
stopped: str = STOP_DONE
|
||||
pending: dict[str, Any] | None = None
|
||||
|
||||
|
||||
async def provider_llm(messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None):
|
||||
"""Default LLM adapter backed by ``backend.ai_chat.chat_completion``."""
|
||||
from backend.ai_chat import chat_completion
|
||||
|
||||
return await chat_completion(messages, tools=tools)
|
||||
|
||||
|
||||
def _truncate(payload: Any) -> Any:
|
||||
"""Truncate an oversized tool result before feeding it back to the model."""
|
||||
serialized = json.dumps(payload, ensure_ascii=False, default=str)
|
||||
if len(serialized) <= MAX_TOOL_RESULT_CHARS:
|
||||
return payload
|
||||
return {"truncated": True, "content": serialized[:MAX_TOOL_RESULT_CHARS]}
|
||||
|
||||
|
||||
def _assistant_tool_message(content: str | None, tool_calls: list[Any]) -> dict[str, Any]:
|
||||
"""Build the OpenAI-style assistant message carrying tool calls."""
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": content or "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": call.name,
|
||||
"arguments": json.dumps(call.arguments, ensure_ascii=False, default=str),
|
||||
},
|
||||
}
|
||||
for call in tool_calls
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _deferred_tool_message(call: Any, reason: str | None = None) -> dict[str, Any]:
|
||||
"""Answer a tool call that was not reached because the run stopped early.
|
||||
|
||||
A single LLM response may carry several tool calls; when the run stops
|
||||
before reaching some of them (tool-call quota), the assistant message still
|
||||
lists *all* of them, so every ``tool_call_id`` must get a tool result
|
||||
before the next LLM call (the OpenAI tool protocol rejects dangling ids).
|
||||
The calls that were not reached get a synthetic ``deferred`` result.
|
||||
|
||||
Note: mutating calls that pause the run for confirmation are no longer
|
||||
deferred — they are batched and applied together on resume (BUG-075); this
|
||||
helper remains for budget stops (BUG-050/BUG-052).
|
||||
"""
|
||||
return {
|
||||
"role": "tool",
|
||||
"tool_call_id": call.id,
|
||||
"name": call.name,
|
||||
"content": json.dumps({
|
||||
"status": "deferred",
|
||||
"reason": reason or (
|
||||
"Not executed: the run stopped before reaching this tool call. "
|
||||
"Re-issue this call if it is still needed."
|
||||
),
|
||||
}, ensure_ascii=False),
|
||||
}
|
||||
|
||||
|
||||
def _action_descriptor(call: Any) -> dict[str, Any]:
|
||||
"""Describe one paused mutating tool call for the confirmation payload.
|
||||
|
||||
A single LLM response may request several mutations (create a folder and
|
||||
the files inside it…). They are batched into one confirmation so the user
|
||||
approves the whole plan in one click (BUG-075). ``step`` reuses the
|
||||
Notion-style label, so the confirmation card reads like the steps block.
|
||||
"""
|
||||
return {
|
||||
"id": call.id,
|
||||
"tool": call.name,
|
||||
"arguments": call.arguments,
|
||||
"step": tool_step_label(call.name, call.arguments),
|
||||
}
|
||||
|
||||
|
||||
def _fallback_summary(executed: list[ToolCallRecord]) -> str:
|
||||
"""Deterministic non-empty answer built from the gathered tool results.
|
||||
|
||||
Used only if the final synthesis call fails or returns nothing, so a turn
|
||||
never ends on an empty message (BUG-052).
|
||||
"""
|
||||
lines: list[str] = []
|
||||
for record in executed:
|
||||
data = record.result
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
for item in (data.get("results") or [])[:5]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
title = item.get("title") or item.get("url") or ""
|
||||
url = item.get("url") or ""
|
||||
lines.append(f"- [{title}]({url})" if url else f"- {title}")
|
||||
if data.get("url") and data.get("text"):
|
||||
title = data.get("title") or data["url"]
|
||||
lines.append(f"- [{title}]({data['url']})")
|
||||
if not lines:
|
||||
return "Je n'ai pas pu produire de réponse à partir des résultats obtenus."
|
||||
unique = list(dict.fromkeys(lines))
|
||||
return "Voici les sources pertinentes trouvées :\n" + "\n".join(unique)
|
||||
|
||||
|
||||
async def _finalize_answer(
|
||||
llm: Callable[..., Any],
|
||||
convo: list[dict[str, Any]],
|
||||
executed: list[ToolCallRecord],
|
||||
steps: list[dict[str, Any]],
|
||||
iterations: int,
|
||||
stopped: str,
|
||||
) -> AgentResult:
|
||||
"""Guarantee a textual answer when the loop stopped before producing one.
|
||||
|
||||
Web research often exhausts the iteration budget while the model is still
|
||||
calling tools; returning ``content=""`` left the conversation with steps and
|
||||
sources but no answer. One final tool-less call asks the model to synthesize
|
||||
the gathered results, and a deterministic source list is used as a last
|
||||
resort (BUG-052).
|
||||
"""
|
||||
content = ""
|
||||
if executed:
|
||||
try:
|
||||
response = await llm(
|
||||
[*convo, {"role": "user", "content": _FINALIZE_INSTRUCTION}], []
|
||||
)
|
||||
content = (response.content or "").strip()
|
||||
except Exception as e:
|
||||
logger.warning(f"Agent final synthesis failed: {e}")
|
||||
if not content:
|
||||
content = _fallback_summary(executed)
|
||||
return AgentResult(
|
||||
content=content,
|
||||
messages=convo,
|
||||
tool_calls=executed,
|
||||
steps=steps,
|
||||
iterations=iterations,
|
||||
stopped=stopped,
|
||||
)
|
||||
|
||||
|
||||
def _execute_confirmed(
|
||||
ctx: ToolContext,
|
||||
confirm_pending: dict[str, Any],
|
||||
convo: list[dict[str, Any]],
|
||||
executed: list[ToolCallRecord],
|
||||
on_tool_call: Callable[[ToolCallRecord], None] | None,
|
||||
) -> None:
|
||||
"""Apply previously-paused mutating tool calls and feed their results back.
|
||||
|
||||
The pending payload is the ``error`` object emitted by a ``confirmation``
|
||||
event, optionally carrying an ``actions`` list with every mutating call of
|
||||
the LLM turn (BUG-075). Each action is applied with a one-shot confirmation
|
||||
and its ``tool_call_id`` answered, keeping the conversation valid for the
|
||||
resumed turn. The assistant tool-call message is expected to already be in
|
||||
``convo`` (it is part of the snapshot returned with the confirmation).
|
||||
"""
|
||||
from backend.ai_chat import ToolCall
|
||||
|
||||
error = confirm_pending.get("error", confirm_pending) or {}
|
||||
actions = confirm_pending.get("actions")
|
||||
if not isinstance(actions, list) or not actions:
|
||||
# Legacy single-action payload (no ``actions`` list).
|
||||
actions = [{
|
||||
"id": error.get("id") or "call_pending",
|
||||
"tool": error.get("tool"),
|
||||
"arguments": error.get("arguments") or {},
|
||||
}]
|
||||
|
||||
for action in actions:
|
||||
name = action.get("tool")
|
||||
arguments = action.get("arguments") or {}
|
||||
call_id = action.get("id") or "call_pending"
|
||||
|
||||
if not name:
|
||||
raise ToolError("Malformed confirmation payload", code="invalid_confirmation")
|
||||
|
||||
# Make sure the assistant tool-call message is present in the snapshot.
|
||||
if not any(
|
||||
m.get("role") == "assistant" and any(
|
||||
tc.get("id") == call_id for tc in (m.get("tool_calls") or [])
|
||||
)
|
||||
for m in convo
|
||||
):
|
||||
convo.append(_assistant_tool_message(None, [ToolCall(id=call_id, name=name, arguments=arguments)]))
|
||||
|
||||
try:
|
||||
result = call_tool(name, ctx, arguments, confirm=True)
|
||||
payload = result.data
|
||||
ok = True
|
||||
except ToolError as e:
|
||||
payload = e.to_dict()
|
||||
ok = False
|
||||
|
||||
record = ToolCallRecord(
|
||||
name=name, arguments=arguments, ok=ok, result=payload,
|
||||
step=tool_step_label(name, arguments),
|
||||
)
|
||||
executed.append(record)
|
||||
if on_tool_call is not None:
|
||||
on_tool_call(record)
|
||||
|
||||
convo.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
"name": name,
|
||||
"content": json.dumps(_truncate(payload), ensure_ascii=False, default=str),
|
||||
})
|
||||
|
||||
|
||||
async def run_agent(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
ctx: ToolContext,
|
||||
llm: Callable[..., Any] | None = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
max_iterations: int = DEFAULT_MAX_ITERATIONS,
|
||||
max_tool_calls: int | None = None,
|
||||
on_tool_call: Callable[[ToolCallRecord], None] | None = None,
|
||||
on_thought: Callable[[dict[str, Any]], None] | None = None,
|
||||
resume_messages: list[dict[str, Any]] | None = None,
|
||||
confirm_pending: dict[str, Any] | None = None,
|
||||
) -> AgentResult:
|
||||
"""Run the tool-calling loop until completion.
|
||||
|
||||
Args:
|
||||
messages: Initial conversation (OpenAI-style), typically a system
|
||||
message followed by the conversation history and the user message.
|
||||
ctx: Tool execution context (identity, mode, confirmation state).
|
||||
llm: Async callable ``(messages, tools) -> LLMResponse``. Defaults to
|
||||
the real provider adapter.
|
||||
tools: Tool schemas to expose. ``None`` exposes all in-app tools;
|
||||
pass ``[]`` to disable tool calling (plain chat).
|
||||
max_iterations: Hard cap on LLM round-trips.
|
||||
max_tool_calls: Hard cap on the total number of executed tool calls
|
||||
(quota, defaults to ``BOOKSLM_MAX_TOOL_CALLS``).
|
||||
on_tool_call: Optional callback invoked after each executed tool call.
|
||||
resume_messages: Conversation snapshot from a paused run (returned with
|
||||
a ``confirmation`` event). When set, the loop resumes from it.
|
||||
confirm_pending: Pending mutating tool call to apply before resuming
|
||||
(two-step propose/apply).
|
||||
|
||||
Returns:
|
||||
An :class:`AgentResult`. ``stopped`` is ``done``, ``max_iterations`` or
|
||||
``confirmation_required`` (in which case ``pending`` holds the payload
|
||||
to confirm, for the two-step propose/apply flow).
|
||||
"""
|
||||
llm = llm or provider_llm
|
||||
if tools is None:
|
||||
tools = get_tool_schemas(scope=ToolScope.IN_APP)
|
||||
quota = DEFAULT_MAX_TOOL_CALLS if max_tool_calls is None else max_tool_calls
|
||||
steps: list[dict[str, Any]] = []
|
||||
|
||||
def _emit_note(text: str) -> None:
|
||||
"""Record an intermediate reasoning note as a visible step."""
|
||||
note = thought_step_label(text)
|
||||
if note["params"]["value"]:
|
||||
steps.append(note)
|
||||
if on_thought is not None:
|
||||
on_thought(note)
|
||||
|
||||
convo = [dict(m) for m in (resume_messages if resume_messages is not None else messages)]
|
||||
executed: list[ToolCallRecord] = []
|
||||
|
||||
def _run_call(call: Any) -> None:
|
||||
"""Execute one tool call, record it and answer its ``tool_call_id``.
|
||||
|
||||
``ToolConfirmationRequired`` propagates to the caller so the loop can
|
||||
pause and batch the mutating calls of the turn (BUG-075).
|
||||
"""
|
||||
try:
|
||||
result = call_tool(call.name, ctx, call.arguments)
|
||||
payload: Any = result.data
|
||||
ok = True
|
||||
except ToolConfirmationRequired:
|
||||
raise
|
||||
except ToolError as e:
|
||||
payload = e.to_dict()
|
||||
ok = False
|
||||
record = ToolCallRecord(
|
||||
name=call.name, arguments=call.arguments, ok=ok, result=payload,
|
||||
step=tool_step_label(call.name, call.arguments),
|
||||
)
|
||||
executed.append(record)
|
||||
steps.append(record.step)
|
||||
if on_tool_call is not None:
|
||||
on_tool_call(record)
|
||||
convo.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": call.id,
|
||||
"name": call.name,
|
||||
"content": json.dumps(_truncate(payload), ensure_ascii=False, default=str),
|
||||
})
|
||||
|
||||
if confirm_pending:
|
||||
if quota is not None and len(executed) >= quota:
|
||||
return AgentResult(
|
||||
content="",
|
||||
messages=convo,
|
||||
tool_calls=executed,
|
||||
steps=steps,
|
||||
iterations=0,
|
||||
stopped=STOP_QUOTA_EXCEEDED,
|
||||
)
|
||||
_execute_confirmed(ctx, confirm_pending, convo, executed, on_tool_call)
|
||||
|
||||
for iteration in range(1, max_iterations + 1):
|
||||
response = await llm(convo, tools)
|
||||
|
||||
if not response.has_tool_calls:
|
||||
return AgentResult(
|
||||
content=response.content or "",
|
||||
messages=convo,
|
||||
tool_calls=executed,
|
||||
steps=steps,
|
||||
iterations=iteration,
|
||||
stopped=STOP_DONE,
|
||||
)
|
||||
|
||||
# Intermediate reasoning shown alongside tool calls → a "thought" step.
|
||||
_emit_note(response.content or "")
|
||||
convo.append(_assistant_tool_message(response.content, response.tool_calls))
|
||||
|
||||
for index, call in enumerate(response.tool_calls):
|
||||
if quota is not None and len(executed) >= quota:
|
||||
logger.warning(f"Agent reached the tool-call quota ({quota})")
|
||||
# Keep the conversation valid for the synthesis call: the
|
||||
# assistant message announced every tool call of the batch.
|
||||
for skipped in response.tool_calls[index:]:
|
||||
convo.append(_deferred_tool_message(
|
||||
skipped, "Not executed: the tool-call quota was reached."
|
||||
))
|
||||
return await _finalize_answer(
|
||||
llm, convo, executed, steps, iteration, STOP_QUOTA_EXCEEDED
|
||||
)
|
||||
try:
|
||||
_run_call(call)
|
||||
except ToolConfirmationRequired as e:
|
||||
logger.info(f"Agent paused: confirmation required for '{call.name}'")
|
||||
pending = e.to_dict()
|
||||
# Include the tool-call id so the client can echo it back.
|
||||
pending["error"]["id"] = call.id
|
||||
# BUG-075: batch every mutating call of this LLM turn so the
|
||||
# user approves the whole plan at once (one resume applies them
|
||||
# all) instead of approving one action after another. Read-only
|
||||
# calls of the batch run immediately and answer their
|
||||
# ``tool_call_id`` so the resumed turn stays valid.
|
||||
actions = [_action_descriptor(call)]
|
||||
for after in response.tool_calls[index + 1:]:
|
||||
spec = get_tool(after.name)
|
||||
if spec is not None and spec.requires_confirmation:
|
||||
actions.append(_action_descriptor(after))
|
||||
else:
|
||||
_run_call(after)
|
||||
pending["actions"] = actions
|
||||
return AgentResult(
|
||||
content=response.content or "",
|
||||
messages=convo,
|
||||
tool_calls=executed,
|
||||
steps=steps,
|
||||
iterations=iteration,
|
||||
stopped=STOP_CONFIRMATION_REQUIRED,
|
||||
pending=pending,
|
||||
)
|
||||
|
||||
logger.warning(f"Agent reached max iterations ({max_iterations})")
|
||||
return await _finalize_answer(
|
||||
llm, convo, executed, steps, max_iterations, STOP_MAX_ITERATIONS
|
||||
)
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Émission d'événements vers le journal du service d'agents
|
||||
(Fusion, Lot 2 vague 5 — contrat ACC-0.2, Roadmap #220).
|
||||
|
||||
ObsiGate émet ses mutations de notes **explicites** vers
|
||||
``POST /api/v1/evenements`` du service d'agents, qui les expose dans
|
||||
le journal de sa console. L'enveloppe ne contient qu'une référence,
|
||||
un libellé et un lien — jamais le contenu de la note :
|
||||
``{id, type, occurred_at, actor, domain, source_app,
|
||||
object{ref, label, url}}``.
|
||||
|
||||
**Garde-fou (conception validée)** : ce module n'est appelé que
|
||||
depuis les points d'écriture explicites de l'API web —
|
||||
:mod:`backend.routers.files_write` (session utilisateur, acteur
|
||||
``user``) et la couche outils :mod:`backend.tools.service`
|
||||
(serveur MCP d'ObsiGate, acteur ``agent:mcp`` ; assistant intégré,
|
||||
acteur ``user``). Le **watcher** (:mod:`backend.watcher`) et
|
||||
l'indexeur ne l'appellent **jamais** : le vault principal compte
|
||||
plus de 220 000 fichiers, un branchement sur les événements du
|
||||
système de fichiers inonderait le journal. Une modification faite
|
||||
hors API (éditeur externe, synchro disque…) reste donc invisible
|
||||
du journal — c'est voulu.
|
||||
|
||||
Authentification : signature HMAC-SHA256 du corps brut, en-tête
|
||||
``X-Draco-Signature``, clé propre à ObsiGate
|
||||
(``AGENT_EVENEMENTS_CLE``, jamais en dur). Sans clé, le module est
|
||||
**inactif** : aucun appel, aucun bruit. URL surchargée par
|
||||
``AGENT_EVENEMENTS_URL``.
|
||||
|
||||
Discipline « au mieux » (même esprit que TikTak et Kanbante) :
|
||||
l'envoi part dans un fil d'arrière-plan démon, délai maximal 2 s,
|
||||
et toute erreur est avalée avec une ligne de journal — une panne
|
||||
du récepteur ne doit jamais casser ni ralentir le métier.
|
||||
|
||||
Domaine : ObsiGate ne modélise pas encore le domaine d'un vault
|
||||
(question Q-A du programme Fusion toujours ouverte). La
|
||||
correspondance peut être déclarée par ``AGENT_EVENEMENTS_DOMAINES``
|
||||
(JSON ``{"NomDuVault": "famille"}``) ; repli ``pro``, qui est aussi
|
||||
le défaut du journal du service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger("obsigate.agent_journal")
|
||||
|
||||
SOURCE_APP = "obsigate"
|
||||
# Défaut officiel, conservé pour référence : depuis #223, ``_url()``
|
||||
# passe par la config centralisée (``backend.draco_config``) — jamais
|
||||
# un nom d'environnement particulier.
|
||||
URL_DEFAUT = "https://agent.dracodev.net/api/v1/evenements"
|
||||
DELAI_S = 2.0
|
||||
|
||||
TYPE_CREE = "obsigate.note.created"
|
||||
TYPE_EDITE = "obsigate.note.edited"
|
||||
TYPE_DEPLACE = "obsigate.note.moved"
|
||||
TYPE_SUPPRIME = "obsigate.note.deleted"
|
||||
|
||||
ACTEUR_USER = "user"
|
||||
ACTEUR_MCP = "agent:mcp"
|
||||
|
||||
|
||||
def _cle() -> str:
|
||||
return os.environ.get("AGENT_EVENEMENTS_CLE", "").strip()
|
||||
|
||||
|
||||
def _url() -> str:
|
||||
# #223 : AGENT_EVENEMENTS_URL d'abord ; à défaut, l'URL du service
|
||||
# configurée (DRACO_AGENT_URL) complétée du chemin d'événements ;
|
||||
# à défaut, le défaut officiel.
|
||||
explicite = os.environ.get("AGENT_EVENEMENTS_URL", "").strip()
|
||||
if explicite:
|
||||
return explicite
|
||||
from backend.draco_config import agent_url
|
||||
|
||||
return f"{agent_url()}/api/v1/evenements"
|
||||
|
||||
|
||||
def actif() -> bool:
|
||||
"""L'émission n'existe que si la clé d'ObsiGate est configurée."""
|
||||
return bool(_cle())
|
||||
|
||||
|
||||
def domaine_du_vault(vault: str) -> str:
|
||||
"""Domaine journal du vault : correspondance déclarée dans
|
||||
``AGENT_EVENEMENTS_DOMAINES`` (JSON), repli ``pro``."""
|
||||
brut = os.environ.get("AGENT_EVENEMENTS_DOMAINES", "").strip()
|
||||
if brut:
|
||||
try:
|
||||
correspondance = json.loads(brut)
|
||||
except ValueError:
|
||||
correspondance = {}
|
||||
if isinstance(correspondance, dict):
|
||||
domaine = correspondance.get(vault)
|
||||
if isinstance(domaine, str) and domaine:
|
||||
return domaine
|
||||
return "pro"
|
||||
|
||||
|
||||
def lien_fichier(vault: str, chemin: str) -> str:
|
||||
"""Lien profond ObsiGate vers un fichier (format ``#file=`` lu
|
||||
par le frontend : ``#file=<vault>:<chemin>``, décodé en URI)."""
|
||||
return f"/#file={quote(vault, safe='')}:{quote(chemin, safe='/')}"
|
||||
|
||||
|
||||
def construire_enveloppe(
|
||||
type_: str,
|
||||
vault: str,
|
||||
chemin: str,
|
||||
actor: str = ACTEUR_USER,
|
||||
) -> tuple[dict, bytes]:
|
||||
"""L'enveloppe ACC-0.2 et son corps JSON exact (celui qui est
|
||||
signé puis envoyé — la signature porte sur ces octets-là).
|
||||
``chemin`` est le chemin relatif au vault ; pour un déplacement,
|
||||
c'est le **nouveau** chemin (la référence pointe sur l'objet tel
|
||||
qu'il existe après l'opération)."""
|
||||
env = {
|
||||
"id": uuid.uuid4().hex,
|
||||
"type": type_,
|
||||
"occurred_at": datetime.now(UTC).isoformat(),
|
||||
"actor": actor or ACTEUR_USER,
|
||||
"domain": domaine_du_vault(vault),
|
||||
"source_app": SOURCE_APP,
|
||||
"object": {
|
||||
"ref": f"fichier:{vault}/{chemin}",
|
||||
"label": chemin.rsplit("/", 1)[-1] if chemin else vault,
|
||||
"url": lien_fichier(vault, chemin),
|
||||
},
|
||||
}
|
||||
corps = json.dumps(env, ensure_ascii=False).encode("utf-8")
|
||||
return env, corps
|
||||
|
||||
|
||||
def _signer(corps: bytes) -> str:
|
||||
return hmac.new(_cle().encode("utf-8"), corps, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _envoyer(corps: bytes) -> None:
|
||||
"""Le POST signé, dans le fil d'arrière-plan. Avale tout : le
|
||||
journal distant est un confort, jamais une dépendance."""
|
||||
try:
|
||||
with httpx.Client(timeout=DELAI_S, trust_env=False) as client:
|
||||
client.post(
|
||||
_url(),
|
||||
content=corps,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Draco-Signature": _signer(corps),
|
||||
},
|
||||
)
|
||||
except Exception as exc: # panne, délai, DNS… : on log et on passe
|
||||
logger.warning("événement non livré au journal du service (%s)", exc)
|
||||
|
||||
|
||||
def emettre(
|
||||
type_: str,
|
||||
vault: str,
|
||||
chemin: str,
|
||||
actor: str = ACTEUR_USER,
|
||||
) -> bool:
|
||||
"""Émet un événement en tâche de fond. Retourne True si l'envoi
|
||||
a été déclenché, False si le module est inactif (pas de clé) ou
|
||||
si l'enveloppe n'a pas pu être construite. Ne lève jamais."""
|
||||
if not actif():
|
||||
return False
|
||||
try:
|
||||
_env, corps = construire_enveloppe(type_, vault, chemin, actor)
|
||||
except Exception:
|
||||
logger.warning("événement %s non construit", type_, exc_info=True)
|
||||
return False
|
||||
threading.Thread(
|
||||
target=_envoyer, args=(corps,), daemon=True, name="obsigate-evenements"
|
||||
).start()
|
||||
return True
|
||||
+99
-13
@@ -1,6 +1,6 @@
|
||||
"""ObsiGate AI — Multi-provider AI service for editor enhancement.
|
||||
|
||||
Supports: DeepSeek, OpenRouter, Google Gemini.
|
||||
Supports: DeepSeek, OpenRouter, Google Gemini, Ollama, NVIDIA, QwenCloud, Xiaomi, Mistral.
|
||||
Configured via environment variables.
|
||||
"""
|
||||
|
||||
@@ -14,10 +14,11 @@ import httpx
|
||||
|
||||
logger = logging.getLogger("obsigate.ai")
|
||||
|
||||
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama"]
|
||||
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama", "nvidia", "qwencloud", "xiaomi", "mistral"]
|
||||
|
||||
# Provider configurations — keys loaded from file or .env
|
||||
AI_KEYS_FILE = Path("data/api_keys.json")
|
||||
APP_CONFIG_FILE = Path(__file__).resolve().parent.parent / "data" / "config.json"
|
||||
|
||||
def _read_ai_keys() -> dict:
|
||||
if not AI_KEYS_FILE.exists():
|
||||
@@ -27,6 +28,16 @@ def _read_ai_keys() -> dict:
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _read_app_config() -> dict:
|
||||
"""Read the persisted application config (``data/config.json``)."""
|
||||
if not APP_CONFIG_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(APP_CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def get_ai_key(env_name: str) -> str:
|
||||
"""Get AI key: stored file first, then .env fallback."""
|
||||
keys = _read_ai_keys()
|
||||
@@ -36,7 +47,7 @@ def get_ai_key(env_name: str) -> str:
|
||||
|
||||
def _load_provider_keys():
|
||||
"""Load AI keys from stored file, falling back to .env."""
|
||||
return {
|
||||
providers = {
|
||||
"deepseek": {
|
||||
"api_key": get_ai_key("DEEPSEEK_API_KEY"),
|
||||
"base_url": "https://api.deepseek.com/v1",
|
||||
@@ -61,18 +72,75 @@ def _load_provider_keys():
|
||||
"model": os.getenv("OLLAMA_MODEL", "qwen2.5-coder:1.5b"),
|
||||
"auth_header": "Bearer {api_key}",
|
||||
},
|
||||
"nvidia": {
|
||||
"api_key": get_ai_key("NVIDIA_API_KEY"),
|
||||
"base_url": "https://integrate.api.nvidia.com/v1",
|
||||
"model": os.getenv("NVIDIA_MODEL", "meta/llama-3.1-405b-instruct"),
|
||||
"auth_header": "Bearer {api_key}",
|
||||
},
|
||||
"qwencloud": {
|
||||
"api_key": get_ai_key("QWENCLOUD_API_KEY"),
|
||||
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
"model": os.getenv("QWENCLOUD_MODEL", "qwen-max"),
|
||||
"auth_header": "Bearer {api_key}",
|
||||
},
|
||||
"xiaomi": {
|
||||
"api_key": get_ai_key("XIAOMI_API_KEY"),
|
||||
"base_url": os.getenv("XIAOMI_BASE_URL", "https://api.xiaomimimo.com/v1"),
|
||||
"model": os.getenv("XIAOMI_MODEL", "mimo-v2.5-pro"),
|
||||
# Xiaomi MiMo uses a dedicated `api-key` header (NOT Authorization: Bearer).
|
||||
# The `_call_deepseek_openrouter` helper substitutes {api_key} verbatim,
|
||||
# so we just emit the raw key value here.
|
||||
"auth_header": "{api_key}",
|
||||
"auth_header_name": "api-key",
|
||||
},
|
||||
"mistral": {
|
||||
"api_key": get_ai_key("MISTRAL_API_KEY"),
|
||||
"base_url": "https://api.mistral.ai/v1",
|
||||
"model": os.getenv("MISTRAL_MODEL", "mistral-large-latest"),
|
||||
"auth_header": "Bearer {api_key}",
|
||||
},
|
||||
}
|
||||
# Apply persisted per-provider model overrides (data/config.json).
|
||||
overrides = _read_app_config().get("ai_default_models") or {}
|
||||
if isinstance(overrides, dict):
|
||||
for name, model in overrides.items():
|
||||
if name in providers and isinstance(model, str) and model:
|
||||
providers[name]["model"] = model
|
||||
return providers
|
||||
|
||||
|
||||
PROVIDERS = _load_provider_keys()
|
||||
|
||||
DEFAULT_PROVIDER: ProviderName = os.getenv("AI_DEFAULT_PROVIDER", "deepseek") # type: ignore
|
||||
|
||||
|
||||
def get_default_provider() -> str:
|
||||
"""Resolve the default provider: ``data/config.json`` > env > ``deepseek``."""
|
||||
provider: str = str(_read_app_config().get("ai_default_provider") or os.getenv("AI_DEFAULT_PROVIDER", "deepseek"))
|
||||
return provider if provider in PROVIDERS else "deepseek"
|
||||
|
||||
|
||||
def reload_ai_config() -> str:
|
||||
"""Reload provider keys and model overrides from disk into ``PROVIDERS`` in place.
|
||||
|
||||
Mutating ``PROVIDERS`` (rather than rebinding it) keeps references held by
|
||||
other modules valid. Returns the resolved default provider.
|
||||
"""
|
||||
global DEFAULT_PROVIDER
|
||||
PROVIDERS.clear()
|
||||
PROVIDERS.update(_load_provider_keys())
|
||||
DEFAULT_PROVIDER = get_default_provider() # type: ignore[assignment]
|
||||
logger.info(f"AI config reloaded (default provider: {DEFAULT_PROVIDER})")
|
||||
return DEFAULT_PROVIDER
|
||||
|
||||
|
||||
def _get_provider_config(provider: ProviderName | None = None) -> dict:
|
||||
"""Get provider config, falling back to default if requested provider unavailable."""
|
||||
p = provider or DEFAULT_PROVIDER
|
||||
default = get_default_provider()
|
||||
p = provider or default
|
||||
if p not in PROVIDERS:
|
||||
p = DEFAULT_PROVIDER
|
||||
p = default
|
||||
cfg = PROVIDERS[p]
|
||||
if not cfg["api_key"]:
|
||||
# Try next available provider
|
||||
@@ -84,17 +152,31 @@ def _get_provider_config(provider: ProviderName | None = None) -> dict:
|
||||
return {"name": p, **cfg}
|
||||
|
||||
|
||||
def _build_headers(cfg: dict) -> dict:
|
||||
"""Build HTTP headers for an OpenAI-compatible provider config.
|
||||
|
||||
Most providers use ``Authorization: Bearer KEY``. Some (Xiaomi MiMo) use a
|
||||
dedicated header like ``api-key: KEY`` — supported via the
|
||||
``auth_header_name`` key in PROVIDERS (defaults to ``Authorization``).
|
||||
"""
|
||||
header_name = cfg.get("auth_header_name") or "Authorization"
|
||||
header_value = cfg["auth_header"].format(api_key=cfg["api_key"])
|
||||
if header_name == "Authorization" and not header_value.lower().startswith("bearer "):
|
||||
header_value = "Bearer " + header_value
|
||||
return {
|
||||
header_name: header_value,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
async def _call_deepseek_openrouter(prompt: str, system: str, provider: ProviderName | None = None,
|
||||
temperature: float = 0.7, max_tokens: int = 2048) -> str:
|
||||
"""Call OpenAI-compatible API (DeepSeek, OpenRouter)."""
|
||||
"""Call OpenAI-compatible API (DeepSeek, OpenRouter, Xiaomi MiMo, etc.)."""
|
||||
cfg = _get_provider_config(provider)
|
||||
# Debug: log masked key to diagnose 401
|
||||
key_preview = cfg["api_key"][:8] + "..." + cfg["api_key"][-4:] if len(cfg["api_key"]) > 12 else "***"
|
||||
logger.info(f"AI call: provider={cfg['name']} model={cfg['model']} key={key_preview}")
|
||||
headers = {
|
||||
"Authorization": cfg["auth_header"].format(api_key=cfg["api_key"]),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
headers = _build_headers(cfg)
|
||||
payload = {
|
||||
"model": cfg["model"],
|
||||
"messages": [
|
||||
@@ -139,6 +221,7 @@ async def ai_complete(prompt: str, provider: ProviderName | None = None) -> str:
|
||||
cfg = _get_provider_config(provider)
|
||||
if cfg["name"] == "gemini":
|
||||
return await _call_gemini(prompt, "You are a helpful assistant.")
|
||||
# All other providers use OpenAI-compatible format
|
||||
return await _call_deepseek_openrouter(prompt, "You are a helpful assistant.", provider)
|
||||
|
||||
|
||||
@@ -270,10 +353,13 @@ async def ai_generate_frontmatter(text: str, provider: ProviderName | None = Non
|
||||
|
||||
|
||||
async def ai_inline_complete(text: str, provider: ProviderName | None = None) -> str:
|
||||
"""Inline completion — suggest continuation."""
|
||||
"""Inline completion — suggest a short continuation of the text before the cursor."""
|
||||
return await _call_deepseek_openrouter(
|
||||
f"Complete this text naturally. Return only the completion (just the new text, no repetition):\n\n{text}",
|
||||
SYSTEM_PROMPT, provider, temperature=0.3, max_tokens=512,
|
||||
"Continue the text below in the same language. Reply with ONLY the "
|
||||
"continuation: no repetition, no quotes, no explanation, at most one "
|
||||
"short sentence. If the text ends with a partial word, finish that word.\n\n"
|
||||
+ text,
|
||||
SYSTEM_PROMPT, provider, temperature=0.2, max_tokens=128,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
"""Provider-agnostic chat completion with native tool (function) calling.
|
||||
|
||||
This module complements ``backend.ai`` (which handles the 16 stateless editor
|
||||
actions). It exposes a single ``chat_completion`` entry point used by the
|
||||
in-app agent loop:
|
||||
|
||||
- OpenAI-compatible providers (DeepSeek, OpenRouter, NVIDIA, QwenCloud,
|
||||
Xiaomi, Mistral) use the ``tools`` / ``tool_calls`` protocol.
|
||||
- Google Gemini uses ``functionDeclarations`` / ``functionCall``.
|
||||
|
||||
When a provider rejects the ``tools`` parameter (model without function
|
||||
calling support), the call is transparently retried without tools so callers
|
||||
degrade to plain chat (fallback protocol, see ``docs/AI_ARCHITECTURE_GUIDE.md``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.ai import PROVIDERS, _build_headers, _get_provider_config
|
||||
|
||||
logger = logging.getLogger("obsigate.ai_chat")
|
||||
|
||||
# Status codes that usually mean "tools not supported by this model".
|
||||
_TOOLS_UNSUPPORTED_STATUS = {400, 404, 422}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCall:
|
||||
"""A single tool invocation requested by the model."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLMResponse:
|
||||
"""Normalized provider response (text and/or tool calls)."""
|
||||
|
||||
content: str | None = None
|
||||
tool_calls: list[ToolCall] = field(default_factory=list)
|
||||
provider: str = ""
|
||||
model: str = ""
|
||||
|
||||
@property
|
||||
def has_tool_calls(self) -> bool:
|
||||
return bool(self.tool_calls)
|
||||
|
||||
|
||||
def _parse_arguments(raw: Any) -> dict[str, Any]:
|
||||
"""Parse tool-call arguments that may arrive as a JSON string or dict."""
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
return parsed if isinstance(parsed, dict) else {"value": parsed}
|
||||
except json.JSONDecodeError:
|
||||
return {"_raw": raw}
|
||||
return {}
|
||||
|
||||
|
||||
async def chat_completion(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 4096,
|
||||
) -> LLMResponse:
|
||||
"""Run a chat completion, optionally with tool calling.
|
||||
|
||||
Args:
|
||||
messages: OpenAI-style messages (``system`` / ``user`` / ``assistant`` /
|
||||
``tool``). Assistant messages may carry ``tool_calls``.
|
||||
tools: OpenAI-style tool schemas (see ``get_tool_schemas``).
|
||||
provider: Provider override; falls back to the configured default.
|
||||
model: Model override; falls back to the provider's default model.
|
||||
temperature: Sampling temperature.
|
||||
max_tokens: Maximum output tokens.
|
||||
|
||||
Returns:
|
||||
:class:`LLMResponse` with the text content and/or requested tool calls.
|
||||
"""
|
||||
cfg = _get_provider_config(provider) # type: ignore[arg-type]
|
||||
name = cfg["name"]
|
||||
resolved_model = model or cfg["model"]
|
||||
|
||||
if name == "gemini":
|
||||
return await _gemini_chat(messages, tools, resolved_model, temperature, max_tokens)
|
||||
return await _openai_chat(messages, tools, cfg, resolved_model, temperature, max_tokens)
|
||||
|
||||
|
||||
async def _openai_chat(
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]] | None,
|
||||
cfg: dict[str, Any],
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
) -> LLMResponse:
|
||||
"""Call an OpenAI-compatible ``/chat/completions`` endpoint."""
|
||||
headers = _build_headers(cfg)
|
||||
url = f"{cfg['base_url']}/chat/completions"
|
||||
|
||||
def _build_payload(use_tools: list[dict[str, Any]] | None) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if use_tools:
|
||||
payload["tools"] = use_tools
|
||||
payload["tool_choice"] = "auto"
|
||||
return payload
|
||||
|
||||
try:
|
||||
data = await _post_json(url, headers, _build_payload(tools))
|
||||
except httpx.HTTPStatusError as e:
|
||||
if tools and e.response.status_code in _TOOLS_UNSUPPORTED_STATUS:
|
||||
logger.warning(f"Provider '{cfg['name']}' rejected tools — retrying without tools")
|
||||
data = await _post_json(url, headers, _build_payload(None))
|
||||
else:
|
||||
raise
|
||||
|
||||
message = data["choices"][0]["message"]
|
||||
content = (message.get("content") or "").strip() or None
|
||||
|
||||
tool_calls: list[ToolCall] = []
|
||||
for idx, tc in enumerate(message.get("tool_calls") or []):
|
||||
fn = tc.get("function", {}) or {}
|
||||
tool_calls.append(ToolCall(
|
||||
id=tc.get("id") or f"call_{idx}",
|
||||
name=fn.get("name", ""),
|
||||
arguments=_parse_arguments(fn.get("arguments")),
|
||||
))
|
||||
|
||||
return LLMResponse(content=content, tool_calls=tool_calls, provider=cfg["name"], model=model)
|
||||
|
||||
|
||||
def _gemini_tools(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]] | None:
|
||||
"""Convert OpenAI-style tool schemas to Gemini ``functionDeclarations``."""
|
||||
if not tools:
|
||||
return None
|
||||
declarations = []
|
||||
for spec in tools:
|
||||
fn = spec.get("function", spec)
|
||||
declarations.append({
|
||||
"name": fn.get("name", ""),
|
||||
"description": fn.get("description", ""),
|
||||
"parameters": fn.get("parameters", {"type": "object", "properties": {}}),
|
||||
})
|
||||
return [{"functionDeclarations": declarations}]
|
||||
|
||||
|
||||
_DATA_URL_RE = re.compile(r"^data:([^;,]+);base64,(.*)$", re.DOTALL)
|
||||
|
||||
|
||||
def _content_to_gemini_parts(content: Any) -> list[dict[str, Any]]:
|
||||
"""Convert OpenAI-style message content to Gemini ``parts``.
|
||||
|
||||
Accepts either a plain string or a multimodal content array
|
||||
(``[{"type": "text", ...}, {"type": "image_url", ...}]``). Data URLs are
|
||||
turned into ``inlineData`` parts so images can be sent to vision models.
|
||||
"""
|
||||
if content is None:
|
||||
return [{"text": ""}]
|
||||
if isinstance(content, str):
|
||||
return [{"text": content}]
|
||||
|
||||
parts: list[dict[str, Any]] = []
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append({"text": item})
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
item_type = item.get("type")
|
||||
if item_type == "text":
|
||||
parts.append({"text": item.get("text", "")})
|
||||
elif item_type == "image_url":
|
||||
url = (item.get("image_url") or {}).get("url", "")
|
||||
match = _DATA_URL_RE.match(url or "")
|
||||
if match:
|
||||
parts.append({
|
||||
"inlineData": {"mimeType": match.group(1), "data": match.group(2)},
|
||||
})
|
||||
elif url:
|
||||
parts.append({"fileData": {"fileUri": url}})
|
||||
if not parts:
|
||||
parts.append({"text": ""})
|
||||
return parts
|
||||
|
||||
|
||||
def _gemini_contents(messages: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
|
||||
"""Split OpenAI-style messages into Gemini ``system`` text + ``contents``."""
|
||||
system_parts: list[str] = []
|
||||
contents: list[dict[str, Any]] = []
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
if role == "system":
|
||||
content = msg.get("content") or ""
|
||||
system_parts.append(content if isinstance(content, str) else "")
|
||||
elif role == "tool":
|
||||
contents.append({
|
||||
"role": "user",
|
||||
"parts": [{
|
||||
"functionResponse": {
|
||||
"name": msg.get("name", ""),
|
||||
"response": {"content": msg.get("content", "")},
|
||||
},
|
||||
}],
|
||||
})
|
||||
else:
|
||||
contents.append({
|
||||
"role": "user" if role == "user" else "model",
|
||||
"parts": _content_to_gemini_parts(msg.get("content")),
|
||||
})
|
||||
return "\n".join(p for p in system_parts if p).strip(), contents
|
||||
|
||||
|
||||
async def _gemini_chat(
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]] | None,
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
) -> LLMResponse:
|
||||
"""Call Gemini's ``generateContent`` endpoint with optional tools."""
|
||||
cfg = PROVIDERS["gemini"]
|
||||
system, contents = _gemini_contents(messages)
|
||||
payload: dict[str, Any] = {
|
||||
"contents": contents,
|
||||
"generationConfig": {"temperature": temperature, "maxOutputTokens": max_tokens},
|
||||
}
|
||||
if system:
|
||||
payload["system_instruction"] = {"parts": [{"text": system}]}
|
||||
gemini_tools = _gemini_tools(tools)
|
||||
if gemini_tools:
|
||||
payload["tools"] = gemini_tools
|
||||
|
||||
url = f"{cfg['base_url']}/models/{model}:generateContent?key={cfg['api_key']}"
|
||||
data = await _post_json(url, None, payload)
|
||||
|
||||
parts = data["candidates"][0]["content"].get("parts", [])
|
||||
content = "".join(p.get("text", "") for p in parts if "text" in p).strip() or None
|
||||
|
||||
tool_calls: list[ToolCall] = []
|
||||
for idx, part in enumerate(parts):
|
||||
fc = part.get("functionCall")
|
||||
if fc:
|
||||
tool_calls.append(ToolCall(
|
||||
id=f"call_{idx}",
|
||||
name=fc.get("name", ""),
|
||||
arguments=_parse_arguments(fc.get("args")),
|
||||
))
|
||||
|
||||
return LLMResponse(content=content, tool_calls=tool_calls, provider="gemini", model=model)
|
||||
|
||||
|
||||
async def _post_json(url: str, headers: dict[str, str] | None, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""POST JSON and return the parsed body, raising on HTTP errors."""
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
resp = await client.post(url, headers=headers or {}, json=payload)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
# ── Streaming (SSE token stream) ────────────────────────────────────────
|
||||
|
||||
|
||||
async def stream_completion(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 4096,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Yield content deltas from a chat completion as they arrive.
|
||||
|
||||
Only text content is streamed (no tool calling): this backs the plain
|
||||
``/api/ai/bookslm/chat`` endpoint. The tool-calling ``/agent`` endpoint
|
||||
keeps using :func:`chat_completion` because tool calls need the complete
|
||||
response before they can be executed.
|
||||
"""
|
||||
cfg = _get_provider_config(provider) # type: ignore[arg-type]
|
||||
resolved_model = model or cfg["model"]
|
||||
|
||||
if cfg["name"] == "gemini":
|
||||
stream = _gemini_stream(messages, resolved_model, temperature, max_tokens)
|
||||
else:
|
||||
stream = _openai_stream(messages, cfg, resolved_model, temperature, max_tokens)
|
||||
|
||||
async for chunk in stream:
|
||||
yield chunk
|
||||
|
||||
|
||||
async def _openai_stream(
|
||||
messages: list[dict[str, Any]],
|
||||
cfg: dict[str, Any],
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream an OpenAI-compatible ``/chat/completions`` response."""
|
||||
headers = _build_headers(cfg)
|
||||
url = f"{cfg['base_url']}/chat/completions"
|
||||
payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client, client.stream("POST", url, headers=headers, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line or not line.startswith("data:"):
|
||||
continue
|
||||
raw = line[5:].strip()
|
||||
if raw == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
choices = chunk.get("choices") or []
|
||||
if not choices:
|
||||
continue
|
||||
delta = choices[0].get("delta") or {}
|
||||
content = delta.get("content")
|
||||
if content:
|
||||
yield content
|
||||
|
||||
|
||||
async def _gemini_stream(
|
||||
messages: list[dict[str, Any]],
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream Gemini's ``streamGenerateContent`` response (SSE)."""
|
||||
cfg = PROVIDERS["gemini"]
|
||||
system, contents = _gemini_contents(messages)
|
||||
payload: dict[str, Any] = {
|
||||
"contents": contents,
|
||||
"generationConfig": {"temperature": temperature, "maxOutputTokens": max_tokens},
|
||||
}
|
||||
if system:
|
||||
payload["system_instruction"] = {"parts": [{"text": system}]}
|
||||
|
||||
url = f"{cfg['base_url']}/models/{model}:streamGenerateContent?alt=sse&key={cfg['api_key']}"
|
||||
async with httpx.AsyncClient(timeout=120.0) as client, client.stream("POST", url, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line or not line.startswith("data:"):
|
||||
continue
|
||||
raw = line[5:].strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
chunk = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for candidate in chunk.get("candidates") or []:
|
||||
for part in candidate.get("content", {}).get("parts", []):
|
||||
text = part.get("text")
|
||||
if text:
|
||||
yield text
|
||||
@@ -0,0 +1,175 @@
|
||||
# backend/ai_history.py
|
||||
"""Persistent assistant conversation history (#95).
|
||||
|
||||
Each authenticated user owns a flat list of conversation sessions tagged with
|
||||
their context (mode / vault / directory / documents). Sessions survive page
|
||||
reloads and are the source of truth for the panel history menu (#96 sidebar
|
||||
"Historique IA" reads the same store).
|
||||
|
||||
Format of a session (JS/JSON shape kept identical to the client, minus
|
||||
transient fields):
|
||||
|
||||
{
|
||||
"id": "s-…",
|
||||
"title": "…",
|
||||
"mode": "directory" | "documents" | "general",
|
||||
"vault": "…" | None,
|
||||
"directory": "…" | "",
|
||||
"documents": [{"vault": "…", "path": "…"}],
|
||||
"context": "directory-…", # _contextKey() of the assistant
|
||||
"createdAt": 1234567890,
|
||||
"updatedAt": 1234567890,
|
||||
"messages": [{"role": "user|assistant", "content": "…"}]
|
||||
}
|
||||
|
||||
Sessions are capped per user (see MAX_SESSIONS); the oldest ones are dropped
|
||||
when the cap is reached.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.ai_history")
|
||||
|
||||
AI_HISTORY_DIR = Path("data/ai_history")
|
||||
MAX_SESSIONS = 200
|
||||
|
||||
|
||||
def _get_user_file(username: str) -> Path:
|
||||
AI_HISTORY_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return AI_HISTORY_DIR / f"{username}.json"
|
||||
|
||||
|
||||
def _read_sessions(username: str) -> list[dict[str, Any]]:
|
||||
path = _get_user_file(username)
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as e: # pragma: no cover - defensive I/O guard
|
||||
logger.error(f"Failed to read AI history for {username}: {e}")
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
return [s for s in data if isinstance(s, dict)]
|
||||
|
||||
|
||||
def _write_sessions(username: str, sessions: list[dict[str, Any]]) -> None:
|
||||
path = _get_user_file(username)
|
||||
try:
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(
|
||||
json.dumps(sessions, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shutil.move(str(tmp), str(path))
|
||||
except Exception as e: # pragma: no cover - defensive I/O guard
|
||||
logger.error(f"Failed to write AI history for {username}: {e}")
|
||||
|
||||
|
||||
def _summary(session: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Compact representation (no messages) used by the list endpoint."""
|
||||
messages = session.get("messages") or []
|
||||
preview = ""
|
||||
for msg in reversed(messages):
|
||||
content = (msg.get("content") or "").strip() if isinstance(msg, dict) else ""
|
||||
if content:
|
||||
preview = content[:120]
|
||||
break
|
||||
return {
|
||||
"id": session.get("id", ""),
|
||||
"title": session.get("title", "") or "",
|
||||
"mode": session.get("mode", "general"),
|
||||
"vault": session.get("vault"),
|
||||
"directory": session.get("directory", ""),
|
||||
"context": session.get("context", ""),
|
||||
"createdAt": session.get("createdAt", 0),
|
||||
"updatedAt": session.get("updatedAt", session.get("createdAt", 0)),
|
||||
"message_count": len(messages),
|
||||
"preview": preview,
|
||||
}
|
||||
|
||||
|
||||
def list_sessions(username: str, *, include_messages: bool = False) -> list[dict[str, Any]]:
|
||||
"""Return the user's sessions, most recently updated first.
|
||||
|
||||
With ``include_messages=False`` (default) a compact summary is returned;
|
||||
the full conversation is fetched per id via :func:`get_session`.
|
||||
"""
|
||||
if not username:
|
||||
return []
|
||||
sessions = sorted(
|
||||
_read_sessions(username),
|
||||
key=lambda s: s.get("updatedAt") or s.get("createdAt") or 0,
|
||||
reverse=True,
|
||||
)
|
||||
if include_messages:
|
||||
return sessions
|
||||
return [_summary(s) for s in sessions]
|
||||
|
||||
|
||||
def get_session(username: str, session_id: str) -> dict[str, Any] | None:
|
||||
if not username or not session_id:
|
||||
return None
|
||||
for session in _read_sessions(username):
|
||||
if session.get("id") == session_id:
|
||||
return session
|
||||
return None
|
||||
|
||||
|
||||
def upsert_session(username: str, session: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Create or update a conversation for the user.
|
||||
|
||||
Returns the stored session, or None when there is no valid id.
|
||||
"""
|
||||
if not username:
|
||||
return None
|
||||
session_id = (session.get("id") or "").strip()
|
||||
if not session_id:
|
||||
return None
|
||||
|
||||
now = session.get("updatedAt") or session.get("createdAt") or 0
|
||||
stored = {
|
||||
"id": session_id,
|
||||
"title": session.get("title", "") or "",
|
||||
"mode": session.get("mode") or "general",
|
||||
"vault": session.get("vault"),
|
||||
"directory": session.get("directory", ""),
|
||||
"documents": session.get("documents") or [],
|
||||
"context": session.get("context", ""),
|
||||
"createdAt": session.get("createdAt") or now,
|
||||
"updatedAt": now,
|
||||
"messages": session.get("messages") or [],
|
||||
}
|
||||
|
||||
sessions = _read_sessions(username)
|
||||
replaced = False
|
||||
for i, existing in enumerate(sessions):
|
||||
if existing.get("id") == session_id:
|
||||
sessions[i] = stored
|
||||
replaced = True
|
||||
break
|
||||
if not replaced:
|
||||
sessions.append(stored)
|
||||
|
||||
sessions.sort(key=lambda s: s.get("updatedAt") or s.get("createdAt") or 0, reverse=True)
|
||||
if len(sessions) > MAX_SESSIONS:
|
||||
logger.info(f"AI history cap reached for {username}: trimming to {MAX_SESSIONS}")
|
||||
sessions = sessions[:MAX_SESSIONS]
|
||||
|
||||
_write_sessions(username, sessions)
|
||||
return stored
|
||||
|
||||
|
||||
def delete_session(username: str, session_id: str) -> bool:
|
||||
if not username or not session_id:
|
||||
return False
|
||||
sessions = _read_sessions(username)
|
||||
remaining = [s for s in sessions if s.get("id") != session_id]
|
||||
if len(remaining) == len(sessions):
|
||||
return False
|
||||
_write_sessions(username, remaining)
|
||||
return True
|
||||
+55
-5
@@ -2,11 +2,10 @@
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from backend.ai import (
|
||||
DEFAULT_PROVIDER,
|
||||
PROVIDERS,
|
||||
ai_change_tone,
|
||||
ai_continue_writing,
|
||||
@@ -24,13 +23,17 @@ from backend.ai import (
|
||||
ai_simplify,
|
||||
ai_summarize,
|
||||
ai_translate,
|
||||
get_default_provider,
|
||||
)
|
||||
from backend.auth.middleware import require_auth
|
||||
from backend.model_capabilities import get_model_capabilities
|
||||
from backend.schemas import AIStatusResponse
|
||||
|
||||
logger = logging.getLogger("obsigate.ai_routes")
|
||||
router = APIRouter(prefix="/api/ai", tags=["AI"])
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
@router.get("/status", response_model=AIStatusResponse)
|
||||
async def api_status():
|
||||
"""Check if AI is configured and which providers are available.
|
||||
|
||||
@@ -42,6 +45,10 @@ async def api_status():
|
||||
"deepseek": "DEEPSEEK_API_KEY",
|
||||
"openrouter": "OPENROUTER_API_KEY",
|
||||
"gemini": "GEMINI_API_KEY",
|
||||
"nvidia": "NVIDIA_API_KEY",
|
||||
"qwencloud": "QWENCLOUD_API_KEY",
|
||||
"xiaomi": "XIAOMI_API_KEY",
|
||||
"mistral": "MISTRAL_API_KEY",
|
||||
}
|
||||
providers = {}
|
||||
for name, env_var in provider_keys.items():
|
||||
@@ -86,7 +93,7 @@ async def api_status():
|
||||
|
||||
return {
|
||||
"configured": any(p["available"] for p in providers.values()),
|
||||
"default_provider": DEFAULT_PROVIDER,
|
||||
"default_provider": get_default_provider(),
|
||||
"providers": providers,
|
||||
"autocomplete": autocomplete,
|
||||
}
|
||||
@@ -97,7 +104,8 @@ class AIRequest(BaseModel):
|
||||
instruction: str | None = Field(None, description="Custom instruction for rewrite")
|
||||
target_lang: str | None = Field(None, description="Target language for translation")
|
||||
tone: str | None = Field(None, description="Target tone (professional, casual, etc.)")
|
||||
provider: str | None = Field(None, description="AI provider override")
|
||||
provider: str | None = Field(None, description="AI provider override (e.g. 'deepseek', 'nvidia')")
|
||||
model: str | None = Field(None, description="Model name override for this request")
|
||||
|
||||
|
||||
class AIResponse(BaseModel):
|
||||
@@ -107,6 +115,12 @@ class AIResponse(BaseModel):
|
||||
|
||||
async def _handle(action, request: AIRequest):
|
||||
"""Wrapper with error handling."""
|
||||
from backend.ai import PROVIDERS
|
||||
# Apply per-request model override (saved/restored around the call)
|
||||
original_model = None
|
||||
if request.model and request.provider and request.provider in PROVIDERS:
|
||||
original_model = PROVIDERS[request.provider].get("model")
|
||||
PROVIDERS[request.provider]["model"] = request.model
|
||||
try:
|
||||
result = await action(request.text, request.provider)
|
||||
return AIResponse(result=result, provider=request.provider or "default")
|
||||
@@ -115,6 +129,9 @@ async def _handle(action, request: AIRequest):
|
||||
except Exception as e:
|
||||
logger.error(f"AI error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"AI service error: {e!s}")
|
||||
finally:
|
||||
if original_model is not None and request.provider in PROVIDERS:
|
||||
PROVIDERS[request.provider]["model"] = original_model
|
||||
|
||||
|
||||
@router.post("/improve", response_model=AIResponse)
|
||||
@@ -217,3 +234,36 @@ async def api_inline_complete(req: AIRequest):
|
||||
async def api_to_canvas(req: AIRequest):
|
||||
"""Convert to Mermaid diagram or outline."""
|
||||
return await _handle(ai_convert_to_canvas, req)
|
||||
|
||||
|
||||
class ModelCapabilitiesResponse(BaseModel):
|
||||
"""Capabilities of a single provider/model pair."""
|
||||
|
||||
provider: str
|
||||
model: str
|
||||
capabilities: dict[str, bool] = Field(
|
||||
description="Flags: chat, embeddings, rerank, images, video, "
|
||||
"audio_speech, audio_transcription, vision",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/model-capabilities", response_model=ModelCapabilitiesResponse)
|
||||
async def api_model_capabilities(
|
||||
provider: str = Query(..., description="Provider identifier"),
|
||||
model: str = Query("", description="Model identifier (optional)"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return the capability flags for a provider/model pair.
|
||||
|
||||
Two layers (BUG-044): flags the provider itself declares in its models
|
||||
endpoint (Mistral ``capabilities``, OpenRouter ``architecture``) win, the
|
||||
curated table in ``backend.model_capabilities`` fills the rest. The
|
||||
declaration snapshot is populated by ``GET /api/config/ai-models``; when it
|
||||
is cold (or the provider declares nothing) the curated table answers alone,
|
||||
so this endpoint never performs a blocking provider call.
|
||||
"""
|
||||
return {
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"capabilities": get_model_capabilities(provider, model),
|
||||
}
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 186 KiB |
@@ -4,10 +4,9 @@ import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.attachment_indexer")
|
||||
from backend.media_types import IMAGE_EXTENSIONS
|
||||
|
||||
# Image file extensions to index
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".bmp", ".ico"}
|
||||
logger = logging.getLogger("obsigate.attachment_indexer")
|
||||
|
||||
# Global attachment index: {vault_name: {filename_lower: [absolute_path, ...]}}
|
||||
attachment_index: dict[str, dict[str, list[Path]]] = {}
|
||||
|
||||
+70
-6
@@ -10,7 +10,7 @@ Format: JSON lines (one JSON object per line) for easy parsing.
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("obsigate.audit")
|
||||
@@ -54,7 +54,7 @@ def log_file_save(
|
||||
):
|
||||
"""Log a file save (PUT) operation."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": "file_save",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
@@ -72,7 +72,7 @@ def log_file_delete(
|
||||
):
|
||||
"""Log a file delete operation."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": "file_delete",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
@@ -88,7 +88,7 @@ def log_config_change(
|
||||
):
|
||||
"""Log a configuration change."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": "config_change",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
@@ -99,7 +99,7 @@ def log_config_change(
|
||||
def log_vault_add(username: str, vault_name: str, vault_path: str, ip: str | None = None):
|
||||
"""Log a vault addition."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": "vault_add",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
@@ -111,7 +111,7 @@ def log_vault_add(username: str, vault_name: str, vault_path: str, ip: str | Non
|
||||
def log_vault_remove(username: str, vault_name: str, ip: str | None = None):
|
||||
"""Log a vault removal."""
|
||||
_write_entry({
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": "vault_remove",
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
@@ -119,6 +119,70 @@ def log_vault_remove(username: str, vault_name: str, ip: str | None = None):
|
||||
})
|
||||
|
||||
|
||||
def log_coffre_event(
|
||||
username: str,
|
||||
action: str,
|
||||
coffre: str | None = None,
|
||||
secret_ref: str | None = None,
|
||||
consommateur: str | None = None,
|
||||
resultat: str | None = None,
|
||||
contexte_id: str | None = None,
|
||||
ip: str | None = None,
|
||||
):
|
||||
"""Journal d'un événement du coffre Secrets (Phase 1b).
|
||||
|
||||
Règle absolue : aucun champ ne contient jamais une valeur de secret,
|
||||
un texte chiffré, un nonce ou un jeton. Seules des références
|
||||
(coffre:<slug>/<nom>), des identifiants et des codes de résultat.
|
||||
"""
|
||||
entry = {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": action,
|
||||
"username": username,
|
||||
"ip": ip or "unknown",
|
||||
}
|
||||
if coffre is not None:
|
||||
entry["coffre"] = coffre
|
||||
if secret_ref is not None:
|
||||
entry["secret_ref"] = secret_ref
|
||||
if consommateur is not None:
|
||||
entry["consommateur"] = consommateur
|
||||
if resultat is not None:
|
||||
entry["resultat"] = resultat
|
||||
if contexte_id is not None:
|
||||
entry["contexte_id"] = contexte_id
|
||||
_write_entry(entry)
|
||||
|
||||
|
||||
def log_compte_admin(
|
||||
acteur: str,
|
||||
action: str,
|
||||
cible: str,
|
||||
ip: str | None = None,
|
||||
details: dict | None = None,
|
||||
):
|
||||
"""Journal d'une action d'administration de compte (HUB-13, fusion #127).
|
||||
|
||||
``acteur`` est l'identité de service appelante (API machine des
|
||||
comptes, en-tête ``X-Draco-Actor``), ``cible`` le compte visé.
|
||||
Règle absolue : ``details`` ne contient jamais de mot de passe, de
|
||||
secret TOTP ni de jeton — seulement des métadonnées (champs
|
||||
modifiés, nombre de jti révoqués…). Le champ ``username`` reprend
|
||||
l'acteur pour rester filtrable par l'écran d'audit existant.
|
||||
"""
|
||||
entry: dict = {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"action": action,
|
||||
"username": acteur,
|
||||
"acteur": acteur,
|
||||
"cible": cible,
|
||||
"ip": ip or "unknown",
|
||||
}
|
||||
if details:
|
||||
entry["details"] = details
|
||||
_write_entry(entry)
|
||||
|
||||
|
||||
def get_recent_entries(limit: int = 100, action: str | None = None) -> list:
|
||||
"""Read the most recent audit log entries.
|
||||
|
||||
|
||||
+233
-32
@@ -7,6 +7,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -23,6 +24,22 @@ ALGORITHM = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_SECONDS = int(os.environ.get("OBSIGATE_ACCESS_TOKEN_TTL", "3600")) # default 1 hour
|
||||
REFRESH_TOKEN_EXPIRE_SECONDS = int(os.environ.get("OBSIGATE_REFRESH_TOKEN_TTL", "604800")) # default 7 days
|
||||
|
||||
#: Persistent API/MCP access tokens (user-managed, shown in the config panel).
|
||||
API_TOKENS_FILE = Path("data/api_tokens.json")
|
||||
#: Accepted values for the expiry selector in the UI (1 day, 1 month, 6 months,
|
||||
#: 1 year, never). "never" → no ``exp`` claim → token valid until revoked.
|
||||
API_TOKEN_EXPIRY_CHOICES = {
|
||||
"1d": 24 * 3600,
|
||||
"30d": 30 * 24 * 3600,
|
||||
"180d": 180 * 24 * 3600,
|
||||
"365d": 365 * 24 * 3600,
|
||||
"never": None,
|
||||
}
|
||||
#: Max active tokens per user (anti hoarding; revoking frees a slot).
|
||||
API_TOKEN_MAX_PER_USER = 50
|
||||
#: AES-GCM key derived once from the JWT secret to encrypt stored tokens.
|
||||
_API_TOKEN_KEY: bytes | None = None
|
||||
|
||||
# In-memory revoked token set (loaded from disk on startup)
|
||||
_revoked_jtis: set = set()
|
||||
_revoked_loaded = False
|
||||
@@ -62,16 +79,21 @@ def create_access_token(user: dict) -> str:
|
||||
return jwt.encode(payload, get_secret_key(), algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def create_refresh_token(username: str) -> tuple:
|
||||
"""Create a JWT refresh token. Returns (token_string, jti)."""
|
||||
def create_refresh_token(username: str, remember: bool = False) -> tuple:
|
||||
"""Create a JWT refresh token. Returns (token_string, jti).
|
||||
|
||||
``remember`` is carried as a claim so token rotation can preserve the
|
||||
30-day vs 7-day lifetime chosen at login.
|
||||
"""
|
||||
now = int(time.time())
|
||||
jti = str(uuid.uuid4())
|
||||
payload = {
|
||||
"sub": username,
|
||||
"jti": jti,
|
||||
"iat": now,
|
||||
"exp": now + REFRESH_TOKEN_EXPIRE_SECONDS,
|
||||
"exp": now + (2592000 if remember else REFRESH_TOKEN_EXPIRE_SECONDS),
|
||||
"type": "refresh",
|
||||
"remember": remember,
|
||||
}
|
||||
return jwt.encode(payload, get_secret_key(), algorithm=ALGORITHM), jti
|
||||
|
||||
@@ -87,48 +109,227 @@ def decode_token(token: str) -> dict | None:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token revocation
|
||||
# ---------------------------------------------------------------------------
|
||||
# The store is a dict {jti: valid_until}: the revocation record may be dropped
|
||||
# once the underlying token's own expiry has passed (by then the JWT is dead
|
||||
# anyway). Long-lived API/MCP tokens (see create_api_token) must therefore be
|
||||
# revoked with their real expiry — a 1-year token revoked last week must not
|
||||
# silently come back to life when a 7-day cleanup purges the record (BUG in
|
||||
# the previous set-based store, fixed with feature #107).
|
||||
|
||||
_revoked_map: dict[str, int] = {}
|
||||
_revoked_loaded = False
|
||||
|
||||
# ROADMAP #85 T10a — verrou autour du read-modify-write du store de
|
||||
# révocation (perte de révocations en cas de logouts concurrents).
|
||||
_revoked_lock = threading.RLock()
|
||||
|
||||
|
||||
def _load_revoked():
|
||||
"""Load revoked token JTIs from disk into memory (once)."""
|
||||
global _revoked_loaded, _revoked_jtis
|
||||
if _revoked_loaded:
|
||||
return
|
||||
if REVOKED_TOKENS_FILE.exists():
|
||||
try:
|
||||
data = json.loads(REVOKED_TOKENS_FILE.read_text())
|
||||
# Clean expired entries (older than 7 days)
|
||||
now = int(time.time())
|
||||
_revoked_jtis = {
|
||||
jti for jti, exp in data.items()
|
||||
if exp > now
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load revoked tokens: {e}")
|
||||
_revoked_jtis = set()
|
||||
_revoked_loaded = True
|
||||
global _revoked_loaded, _revoked_map
|
||||
with _revoked_lock:
|
||||
if _revoked_loaded:
|
||||
return
|
||||
if REVOKED_TOKENS_FILE.exists():
|
||||
try:
|
||||
data = json.loads(REVOKED_TOKENS_FILE.read_text())
|
||||
# Drop entries whose underlying token has itself expired.
|
||||
now = int(time.time())
|
||||
_revoked_map = {
|
||||
jti: int(exp) for jti, exp in data.items()
|
||||
if int(exp) > now
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load revoked tokens: {e}")
|
||||
_revoked_map = {}
|
||||
_revoked_loaded = True
|
||||
|
||||
|
||||
def _save_revoked():
|
||||
"""Persist revoked JTIs to disk."""
|
||||
"""Persist revoked JTIs to disk with their per-token expiry."""
|
||||
REVOKED_TOKENS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Store with expiry timestamp for cleanup
|
||||
now = int(time.time())
|
||||
# Keep entries for 7 days max
|
||||
data = {jti: now + REFRESH_TOKEN_EXPIRE_SECONDS for jti in _revoked_jtis}
|
||||
tmp = REVOKED_TOKENS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data))
|
||||
tmp.write_text(json.dumps(_revoked_map))
|
||||
tmp.replace(REVOKED_TOKENS_FILE)
|
||||
|
||||
|
||||
def revoke_token(jti: str):
|
||||
"""Add a token JTI to the revocation list."""
|
||||
_load_revoked()
|
||||
_revoked_jtis.add(jti)
|
||||
_save_revoked()
|
||||
def revoke_token(jti: str, expires_at: int | None = None):
|
||||
"""Add a token JTI to the revocation list.
|
||||
|
||||
``expires_at`` is the revoked token's own ``exp`` (unix seconds) — the
|
||||
record is kept at least that long so a long-lived API token cannot
|
||||
outlive its revocation. ``None`` means the token never expires (API/MCP
|
||||
"sans fin") → the record is kept forever (capped at ~100 years, the JWT
|
||||
store's practical infinity). Default keeps 7 days (session tokens).
|
||||
"""
|
||||
with _revoked_lock:
|
||||
_load_revoked()
|
||||
now = int(time.time())
|
||||
if expires_at is None:
|
||||
until = now + 100 * 365 * 24 * 3600
|
||||
else:
|
||||
until = max(int(expires_at), now + REFRESH_TOKEN_EXPIRE_SECONDS)
|
||||
_revoked_map[jti] = until
|
||||
_save_revoked()
|
||||
logger.debug(f"Revoked token JTI: {jti[:8]}...")
|
||||
|
||||
|
||||
def is_token_revoked(jti: str) -> bool:
|
||||
"""Check if a token JTI has been revoked."""
|
||||
_load_revoked()
|
||||
return jti in _revoked_jtis
|
||||
with _revoked_lock:
|
||||
_load_revoked()
|
||||
return jti in _revoked_map
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API / MCP tokens (feature #107)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Long-lived access tokens the user creates from the config panel. They are
|
||||
# plain HS256 access-type JWTs (``api: true`` claim), so they authenticate
|
||||
# against BOTH the REST API and the MCP endpoint (/mcp) — which share
|
||||
# ``get_current_user``. The raw token is shown exactly once at creation; the
|
||||
# store keeps metadata only (name, owner, expiry, last use) — no secret
|
||||
# material is written to disk.
|
||||
#
|
||||
# File: data/api_tokens.json
|
||||
# {"version": 1, "tokens": {jti: {name, username, created_at, expires_at, last_used_at}}}
|
||||
|
||||
_api_tokens_lock = threading.RLock()
|
||||
_touch_last_write: dict[str, float] = {}
|
||||
|
||||
|
||||
def _load_api_tokens() -> dict:
|
||||
if not API_TOKENS_FILE.exists():
|
||||
return {"version": 1, "tokens": {}}
|
||||
try:
|
||||
return json.loads(API_TOKENS_FILE.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.error(f"Failed to read api_tokens.json: {e}")
|
||||
return {"version": 1, "tokens": {}}
|
||||
|
||||
|
||||
def _save_api_tokens(data: dict):
|
||||
API_TOKENS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = API_TOKENS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(API_TOKENS_FILE)
|
||||
|
||||
|
||||
#: Endpoint perimeters accepted for scoped API tokens (#199). ``"full"``
|
||||
#: keeps today's behaviour ; ``"files"`` restricts the token to the file
|
||||
#: read/write surface (enforced by ``TokenScopeMiddleware`` on REST and by
|
||||
#: the tool registry on MCP/agent calls).
|
||||
API_TOKEN_SCOPES = ("full", "files")
|
||||
|
||||
|
||||
def create_api_token(
|
||||
user: dict,
|
||||
name: str,
|
||||
expiry_key: str,
|
||||
vaults: list[str] | None = None,
|
||||
scope: str = "full",
|
||||
) -> tuple[dict, str]:
|
||||
"""Create a persistent API/MCP token. Returns (record, jwt_string).
|
||||
|
||||
``expiry_key`` must be one of API_TOKEN_EXPIRY_CHOICES; "never" omits the
|
||||
``exp`` claim (valid until explicitly revoked).
|
||||
|
||||
#199 (synchro TikTak) — optional least-privilege restriction:
|
||||
``vaults`` limits the token to the given vaults (``None`` = snapshot of
|
||||
the creator's vaults, i.e. today's behaviour) and ``scope`` limits the
|
||||
endpoint perimeter (``"full"`` or ``"files"``). Both are embedded in the
|
||||
JWT so enforcement stays stateless; the record keeps them for display.
|
||||
"""
|
||||
if expiry_key not in API_TOKEN_EXPIRY_CHOICES:
|
||||
raise ValueError("Expiration invalide")
|
||||
if scope not in API_TOKEN_SCOPES:
|
||||
raise ValueError("Portée invalide (full | files)")
|
||||
seconds = API_TOKEN_EXPIRY_CHOICES[expiry_key]
|
||||
resolved_vaults = list(vaults) if vaults is not None else list(user.get("vaults", []))
|
||||
with _api_tokens_lock:
|
||||
data = _load_api_tokens()
|
||||
tokens = data["tokens"]
|
||||
mine = sum(1 for t in tokens.values() if t["username"] == user["username"])
|
||||
if mine >= API_TOKEN_MAX_PER_USER:
|
||||
raise ValueError(f"Maximum {API_TOKEN_MAX_PER_USER} tokens par utilisateur")
|
||||
now = int(time.time())
|
||||
jti = str(uuid.uuid4())
|
||||
payload = {
|
||||
"sub": user["username"],
|
||||
"role": user.get("role", "user"),
|
||||
"vaults": resolved_vaults,
|
||||
"jti": jti,
|
||||
"iat": now,
|
||||
"type": "access",
|
||||
"api": True,
|
||||
"scope": scope,
|
||||
}
|
||||
if seconds is not None:
|
||||
payload["exp"] = now + seconds
|
||||
token = jwt.encode(payload, get_secret_key(), algorithm=ALGORITHM)
|
||||
record = {
|
||||
"jti": jti,
|
||||
"name": name[:64] or "API token",
|
||||
"username": user["username"],
|
||||
"created_at": now,
|
||||
"expires_at": payload.get("exp"),
|
||||
"expiry_key": expiry_key,
|
||||
"last_used_at": None,
|
||||
"vaults": resolved_vaults,
|
||||
"scope": scope,
|
||||
}
|
||||
tokens[jti] = record
|
||||
_save_api_tokens(data)
|
||||
return record, token
|
||||
|
||||
|
||||
def list_api_tokens(username: str) -> list[dict]:
|
||||
"""Token metadata for one user, newest first."""
|
||||
data = _load_api_tokens()
|
||||
now = int(time.time())
|
||||
items = []
|
||||
for t in data["tokens"].values():
|
||||
if t["username"] != username:
|
||||
continue
|
||||
# #199 : les clés créées avant la portée limitée n'ont ni scope ni
|
||||
# vaults stockés → portée pleine, comportement inchangé.
|
||||
entry = {**t, "expired": t.get("expires_at") is not None and t["expires_at"] < now}
|
||||
entry.setdefault("scope", "full")
|
||||
entry.setdefault("vaults", None)
|
||||
items.append(entry)
|
||||
return sorted(items, key=lambda t: t["created_at"], reverse=True)
|
||||
|
||||
|
||||
def delete_api_token(jti: str, username: str) -> dict:
|
||||
"""Revoke and remove an API token. Raises KeyError when unknown/not owned."""
|
||||
with _api_tokens_lock:
|
||||
data = _load_api_tokens()
|
||||
record = data["tokens"].get(jti)
|
||||
if not record or record["username"] != username:
|
||||
raise KeyError(jti)
|
||||
# Revoke by jti so the presented JWT stops working even though it is
|
||||
# stateless — kept until its natural expiry (no-expiry → forever).
|
||||
revoke_token(jti, record.get("expires_at"))
|
||||
del data["tokens"][jti]
|
||||
_save_api_tokens(data)
|
||||
return record
|
||||
|
||||
|
||||
def maybe_touch_api_token(jti: str | None, created_or_expires: bool = False):
|
||||
"""Record last usage of an API token, throttled to one disk write/hour."""
|
||||
if not jti:
|
||||
return
|
||||
now = time.time()
|
||||
if now - _touch_last_write.get(jti, 0) < 3600:
|
||||
return
|
||||
_touch_last_write[jti] = now
|
||||
try:
|
||||
with _api_tokens_lock:
|
||||
data = _load_api_tokens()
|
||||
record = data["tokens"].get(jti)
|
||||
if record is None:
|
||||
return
|
||||
record["last_used_at"] = int(now)
|
||||
_save_api_tokens(data)
|
||||
except Exception as e: # never fail an authenticated request over stats
|
||||
logger.debug(f"api_token touch failed: {e}")
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# backend/auth/mfa.py
|
||||
# Multi-Factor Authentication: TOTP + recovery codes.
|
||||
# TOTP via pyotp, recovery codes hashed with argon2 for single-use storage.
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
import pyotp
|
||||
|
||||
logger = logging.getLogger("obsigate.auth.mfa")
|
||||
|
||||
TOTP_ISSUER = "ObsiGate"
|
||||
|
||||
|
||||
def generate_secret() -> str:
|
||||
"""Generate a new TOTP secret (base32-encoded, 160 bits)."""
|
||||
return pyotp.random_base32()
|
||||
|
||||
|
||||
def generate_qr_uri(secret: str, username: str, issuer: str = TOTP_ISSUER) -> str:
|
||||
"""Generate an otpauth:// URI for QR code generation."""
|
||||
totp = pyotp.TOTP(secret)
|
||||
return totp.provisioning_uri(name=username, issuer_name=issuer)
|
||||
|
||||
|
||||
def verify_totp(secret: str, code: str) -> bool:
|
||||
"""Verify a TOTP code with a ±1 window tolerance."""
|
||||
totp = pyotp.TOTP(secret)
|
||||
return totp.verify(code, valid_window=1)
|
||||
|
||||
|
||||
def generate_recovery_codes(n: int = 8) -> list[str]:
|
||||
"""Generate n human-readable recovery codes (XXXX-XXXX format)."""
|
||||
codes = []
|
||||
for _ in range(n):
|
||||
# 8 chars alphanumeric, grouped with dash for readability
|
||||
raw = secrets.token_hex(4).upper()
|
||||
code = f"{raw[:4]}-{raw[4:]}"
|
||||
codes.append(code)
|
||||
return codes
|
||||
|
||||
|
||||
def hash_recovery_code(code: str) -> str:
|
||||
"""Hash a recovery code for storage (SHA-256 for fast comparison).
|
||||
|
||||
We use SHA-256 instead of argon2 here because recovery codes are
|
||||
high-entropy random strings, not user-chosen passwords.
|
||||
"""
|
||||
return hashlib.sha256(code.upper().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def verify_recovery_code(code: str, hashed_codes: list[str]) -> int | None:
|
||||
"""Verify a recovery code against stored hashes.
|
||||
|
||||
Returns the index of the matched code (for removal), or None if invalid.
|
||||
Comparison is case-insensitive.
|
||||
"""
|
||||
code_hash = hash_recovery_code(code)
|
||||
for i, stored_hash in enumerate(hashed_codes):
|
||||
if secrets.compare_digest(code_hash, stored_hash):
|
||||
return i
|
||||
return None
|
||||
+132
-5
@@ -4,17 +4,24 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from fastapi import Depends, HTTPException, Request
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
|
||||
from .jwt_handler import decode_token
|
||||
from backend import draco_session
|
||||
from backend.services.net import get_client_ip
|
||||
|
||||
from .jwt_handler import decode_token, is_token_revoked, maybe_touch_api_token
|
||||
from .user_store import get_user
|
||||
|
||||
logger = logging.getLogger("obsigate.auth.middleware")
|
||||
|
||||
security = HTTPBearer(auto_error=False)
|
||||
|
||||
#: Hosts considered safe to bind without authentication (loopback only).
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost", "0:0:0:0:0:0:0:1"}
|
||||
|
||||
|
||||
def is_auth_enabled() -> bool:
|
||||
"""Check if authentication is enabled via environment variable.
|
||||
@@ -24,6 +31,34 @@ def is_auth_enabled() -> bool:
|
||||
return os.environ.get("OBSIGATE_AUTH_ENABLED", "true").lower() != "false"
|
||||
|
||||
|
||||
def is_insecure_mode_allowed() -> bool:
|
||||
"""True when the operator explicitly accepts running without auth (BUG-037)."""
|
||||
return os.environ.get("OBSIGATE_ALLOW_INSECURE", "false").lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def bind_host_from_argv(argv: list[str] | None = None) -> str | None:
|
||||
"""Extract the ``--host`` value from the process arguments (uvicorn), if any.
|
||||
|
||||
Returns ``None`` when no explicit host is passed (uvicorn then defaults to
|
||||
loopback ``127.0.0.1``).
|
||||
"""
|
||||
args = sys.argv if argv is None else argv
|
||||
for i, arg in enumerate(args):
|
||||
if arg == "--host" and i + 1 < len(args):
|
||||
return args[i + 1]
|
||||
if arg.startswith("--host="):
|
||||
return arg.split("=", 1)[1]
|
||||
return None
|
||||
|
||||
|
||||
def is_loopback_host(host: str | None) -> bool:
|
||||
"""True when *host* is a loopback address (or unset → uvicorn default)."""
|
||||
if not host:
|
||||
return True
|
||||
normalized = host.strip().strip("[]").lower()
|
||||
return normalized in _LOOPBACK_HOSTS
|
||||
|
||||
|
||||
def get_current_user(
|
||||
request: Request,
|
||||
credentials: HTTPAuthorizationCredentials | None = Depends(security),
|
||||
@@ -42,6 +77,7 @@ def get_current_user(
|
||||
"vaults": ["*"],
|
||||
"active": True,
|
||||
"_token_vaults": ["*"],
|
||||
"_request_ip": get_client_ip(request),
|
||||
}
|
||||
|
||||
token = None
|
||||
@@ -51,24 +87,80 @@ def get_current_user(
|
||||
token = request.cookies["access_token"]
|
||||
|
||||
if not token:
|
||||
return None
|
||||
return _draco_fallback(request)
|
||||
|
||||
payload = decode_token(token)
|
||||
if not payload or payload.get("type") != "access":
|
||||
return None
|
||||
return _draco_fallback(request)
|
||||
|
||||
# BUG-027: access tokens revoked at logout must be rejected immediately.
|
||||
# (Le repli draco reste sûr : la déconnexion globale révoque aussi
|
||||
# le jti draco et efface son cookie.)
|
||||
jti = payload.get("jti")
|
||||
if jti and is_token_revoked(jti):
|
||||
return _draco_fallback(request)
|
||||
|
||||
user = get_user(payload["sub"])
|
||||
if not user or not user.get("active"):
|
||||
return None
|
||||
return _draco_fallback(request)
|
||||
|
||||
# BUG-028: a password change invalidates every token issued before it.
|
||||
pca = user.get("password_changed_at")
|
||||
iat = payload.get("iat")
|
||||
if pca is not None and iat is not None:
|
||||
try:
|
||||
if int(iat) < int(float(pca)):
|
||||
return _draco_fallback(request)
|
||||
except (TypeError, ValueError):
|
||||
return _draco_fallback(request)
|
||||
|
||||
# Attach vault permissions from the token (snapshot at login time)
|
||||
user["_token_vaults"] = payload.get("vaults", [])
|
||||
# Attach the token id for per-token rate limiting (AI tool layer).
|
||||
user["_token_jti"] = payload.get("jti")
|
||||
# #199 : périmètre d'endpoints du token (défaut "full" = comportement
|
||||
# actuel, y compris pour les clés créées avant la portée limitée).
|
||||
user["_token_scope"] = payload.get("scope", "full")
|
||||
# Feature #107: track last usage of user-managed API/MCP tokens
|
||||
# (throttled write — this dependency runs on both REST and /mcp paths).
|
||||
if payload.get("api"):
|
||||
maybe_touch_api_token(payload.get("jti"))
|
||||
# BUG-030: expose the real client IP to the audit log.
|
||||
user["_request_ip"] = get_client_ip(request)
|
||||
return user
|
||||
|
||||
|
||||
def require_auth(current_user=Depends(get_current_user)):
|
||||
def _draco_fallback(request: Request) -> dict | None:
|
||||
"""Fusion ACC-3.2 (#9) : à défaut de jeton natif valide, le cookie
|
||||
partagé draco_session authentifie l'utilisateur (sub = id
|
||||
ObsiGate, correspondance directe — voir draco_session.user_for_draco).
|
||||
"""
|
||||
user = draco_session.user_for_draco(request)
|
||||
if user is not None:
|
||||
user["_request_ip"] = get_client_ip(request)
|
||||
return user
|
||||
|
||||
|
||||
def require_auth(request: Request, current_user=Depends(get_current_user)):
|
||||
"""Dependency: require a valid authenticated user."""
|
||||
if not current_user:
|
||||
# HUB-19 (#228) : la session draco a été refusée parce que le
|
||||
# registre des rôles du Hub attribue le niveau « aucun » à ce
|
||||
# compte pour ObsiGate — réponse explicite, distincte du 401
|
||||
# générique. L'authentification native n'est pas concernée :
|
||||
# ce refus ne naît que sur le chemin draco (user_for_draco).
|
||||
if getattr(request.state, "role_hub_refuse", None) == "aucun":
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=(
|
||||
"Accès refusé : le registre des rôles du Hub attribue "
|
||||
"le niveau « aucun » à votre compte pour ObsiGate, "
|
||||
"l'accès par la session draco est donc refusé. "
|
||||
"L'administrateur de la suite peut modifier ce niveau "
|
||||
"dans la matrice des rôles du Hub. L'accès natif "
|
||||
"d'ObsiGate (son propre identifiant) reste possible."
|
||||
),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Authentification requise",
|
||||
@@ -84,6 +176,36 @@ def require_admin(current_user=Depends(require_auth)):
|
||||
return current_user
|
||||
|
||||
|
||||
def require_admin_hub(current_user=Depends(require_admin)):
|
||||
"""Dependency: admin + garde du registre des rôles du Hub (HUB-19, #228).
|
||||
|
||||
Pour les routes d'administration des comptes (``/api/auth/admin/*``) :
|
||||
quand la requête est authentifiée par le **canal draco** (``_draco_sub``
|
||||
présent) et que le rôle du Hub est **connu** (``_hub_role`` non None),
|
||||
le niveau ``administrateur`` du registre est exigé — le rôle local ne
|
||||
suffit plus. Session native, ou rôle inconnu (Hub injoignable) :
|
||||
règles actuelles inchangées.
|
||||
"""
|
||||
if current_user.get("_draco_sub"):
|
||||
role_hub = current_user.get("_hub_role")
|
||||
if role_hub is not None and role_hub != "administrateur":
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=(
|
||||
"Accès réservé au niveau « administrateur » d'ObsiGate "
|
||||
"dans le registre des rôles du Hub (votre niveau actuel : "
|
||||
f"« {role_hub} »). L'administrateur de la suite peut "
|
||||
"l'accorder dans la matrice des rôles du Hub."
|
||||
),
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
def is_home_vault(vault_name: str) -> bool:
|
||||
"""Un dossier personnel (#194) : vault « home-<user> »."""
|
||||
return vault_name.startswith("home-")
|
||||
|
||||
|
||||
def check_vault_access(vault_name: str, user: dict) -> bool:
|
||||
"""Check if a user has access to a specific vault.
|
||||
|
||||
@@ -91,8 +213,13 @@ def check_vault_access(vault_name: str, user: dict) -> bool:
|
||||
- vaults == ["*"] → full access (admin default)
|
||||
- vault_name in vaults → access granted
|
||||
- otherwise → denied
|
||||
|
||||
#194 : un dossier personnel n'est **jamais** couvert par ``*`` — sinon
|
||||
un admin (``vaults: ["*"]``) verrait le dossier de chaque utilisateur.
|
||||
"""
|
||||
vaults = user.get("_token_vaults") or user.get("vaults", [])
|
||||
if is_home_vault(vault_name):
|
||||
return vault_name in vaults
|
||||
if "*" in vaults:
|
||||
return True
|
||||
return vault_name in vaults
|
||||
|
||||
@@ -1,18 +1,51 @@
|
||||
# backend/auth/password.py
|
||||
# Argon2id password hashing — OWASP 2024 recommended algorithm.
|
||||
# Parameters: time_cost=2, memory_cost=64MB, parallelism=2
|
||||
# Parameters (BUG-038): time_cost=2, memory_cost=19 MiB, parallelism=1
|
||||
# (OWASP current recommendation for Argon2id). The previous 64 MiB setting
|
||||
# allowed memory exhaustion under concurrent login attempts.
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, VerifyMismatchError
|
||||
|
||||
#: Argon2id cost parameters (OWASP 2024: m=19456 KiB, t=2, p=1).
|
||||
ARGON2_TIME_COST = 2
|
||||
ARGON2_MEMORY_COST_KIB = 19456 # 19 MiB
|
||||
ARGON2_PARALLELISM = 1
|
||||
|
||||
ph = PasswordHasher(
|
||||
time_cost=2,
|
||||
memory_cost=65536, # 64 MB
|
||||
parallelism=2,
|
||||
time_cost=ARGON2_TIME_COST,
|
||||
memory_cost=ARGON2_MEMORY_COST_KIB,
|
||||
parallelism=ARGON2_PARALLELISM,
|
||||
hash_len=32,
|
||||
salt_len=16,
|
||||
)
|
||||
|
||||
# Password policy (BUG-028). Applied by the API validators at account creation
|
||||
# and password change so the rules stay consistent across both paths.
|
||||
MIN_PASSWORD_LENGTH = 8
|
||||
MAX_PASSWORD_LENGTH = 128
|
||||
|
||||
|
||||
def validate_password_strength(password: str) -> str:
|
||||
"""Validate a plaintext password against the project policy.
|
||||
|
||||
Args:
|
||||
password: Candidate password.
|
||||
|
||||
Returns:
|
||||
The password unchanged when valid.
|
||||
|
||||
Raises:
|
||||
ValueError: When the password is too short, too long or blank.
|
||||
"""
|
||||
if password is None or len(password) < MIN_PASSWORD_LENGTH:
|
||||
raise ValueError(f"Minimum {MIN_PASSWORD_LENGTH} caractères")
|
||||
if len(password) > MAX_PASSWORD_LENGTH:
|
||||
raise ValueError(f"Maximum {MAX_PASSWORD_LENGTH} caractères")
|
||||
if not password.strip():
|
||||
raise ValueError("Le mot de passe ne peut pas être vide")
|
||||
return password
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a password with Argon2id."""
|
||||
|
||||
+820
-56
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,141 @@
|
||||
# backend/auth/scope.py
|
||||
# Scoped API/MCP tokens (#199, synchro TikTak — Gitea #2).
|
||||
#
|
||||
# A token may carry an endpoint perimeter in its ``scope`` JWT claim:
|
||||
# - ``"full"`` (default, and the case of every token issued before #199):
|
||||
# no restriction — today's behaviour, unchanged ;
|
||||
# - ``"files"``: file read/write surface only (vault file CRUD, listings,
|
||||
# search, backups read/restore). Everything else (admin, config, AI,
|
||||
# webhooks, shares, auth management…) answers **403**.
|
||||
#
|
||||
# Vault restriction is orthogonal and needs no new machinery: a scoped token
|
||||
# embeds the allowed vault list in its ``vaults`` JWT claim, so the existing
|
||||
# ``check_vault_access`` (REST routes) and ``ToolContext.require_vault_access``
|
||||
# (MCP/agent tools) refuse other vaults with 403 — for REST *and* MCP alike.
|
||||
"""Endpoint perimeter for scoped API tokens."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
SCOPE_FULL = "full"
|
||||
SCOPE_FILES = "files"
|
||||
KNOWN_SCOPES = (SCOPE_FULL, SCOPE_FILES)
|
||||
|
||||
# (methods, path prefix or exact path). ``methods`` None = any method.
|
||||
# Keep this list tight: a sync client (TikTak) needs file CRUD, listings,
|
||||
# search and backup read/restore — nothing else.
|
||||
_RULES_FOR_FILES: tuple[tuple[frozenset[str] | None, str, bool], ...] = (
|
||||
# (methods, path, exact_only)
|
||||
(frozenset({"GET"}), "/api/health", False),
|
||||
(frozenset({"GET"}), "/api/auth/me", True),
|
||||
(frozenset({"GET"}), "/api/vaults", True),
|
||||
(None, "/api/browse/", False),
|
||||
(None, "/api/file/", False),
|
||||
(None, "/api/media/", False),
|
||||
(None, "/api/directory/", False),
|
||||
(None, "/api/move/", False),
|
||||
(None, "/api/vault/", False), # {vault}/files, {vault}/paths, batch-upload…
|
||||
(frozenset({"GET"}), "/api/search", False),
|
||||
(frozenset({"GET"}), "/api/tree-search", True),
|
||||
(frozenset({"GET"}), "/api/suggest", False),
|
||||
(frozenset({"GET"}), "/api/tags", False),
|
||||
(frozenset({"GET"}), "/api/graph/", False),
|
||||
(frozenset({"GET"}), "/api/backups", False),
|
||||
(frozenset({"GET"}), "/docs", False),
|
||||
(frozenset({"GET"}), "/redoc", True),
|
||||
(frozenset({"GET"}), "/openapi.json", True),
|
||||
(None, "/mcp", False),
|
||||
)
|
||||
|
||||
|
||||
def is_path_allowed_for_scope(scope: str | None, method: str, path: str) -> bool:
|
||||
"""True when a token with *scope* may call (*method*, *path*).
|
||||
|
||||
- ``None``/``"full"`` → everything allowed (backward compatible).
|
||||
- ``"files"`` → allow-list above (API surface only ; static assets and
|
||||
public pages carry no data and are handled by the caller — the
|
||||
middleware only gates ``/api/*`` and ``/mcp*``).
|
||||
- unknown scope → fail-closed (deny).
|
||||
"""
|
||||
if scope in (None, SCOPE_FULL):
|
||||
return True
|
||||
if scope != SCOPE_FILES:
|
||||
return False
|
||||
if not (path.startswith("/api/") or path.startswith("/mcp")):
|
||||
return True
|
||||
method = (method or "GET").upper()
|
||||
for methods, rule_path, exact in _RULES_FOR_FILES:
|
||||
if methods is not None and method not in methods:
|
||||
continue
|
||||
if exact:
|
||||
if path == rule_path:
|
||||
return True
|
||||
elif path == rule_path or path.startswith(rule_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def scope_of_user(user: dict[str, Any] | None) -> str:
|
||||
"""Endpoint scope attached to an authenticated user (default ``"full"``)."""
|
||||
if not user:
|
||||
return SCOPE_FULL
|
||||
return user.get("_token_scope") or SCOPE_FULL
|
||||
|
||||
|
||||
class TokenScopeMiddleware:
|
||||
"""Pure-ASGI middleware enforcing the endpoint perimeter of scoped tokens.
|
||||
|
||||
Denies out-of-perimeter calls made with a scoped token (**403**) before
|
||||
they reach the routes. Only *adds* restrictions: the ``scope`` claim is
|
||||
read unverified here, and the signature is still verified downstream by
|
||||
``get_current_user`` (an invalid token is rejected with 401 a step later
|
||||
anyway). Full-scope and scopeless tokens pass through untouched.
|
||||
"""
|
||||
|
||||
def __init__(self, app: Any):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
|
||||
if scope.get("type") == "http":
|
||||
method = scope.get("method", "GET")
|
||||
path = scope.get("path", "")
|
||||
if path.startswith("/api/") or path.startswith("/mcp"):
|
||||
token_scope = _scope_from_asgi(scope)
|
||||
if token_scope not in (None, SCOPE_FULL) and not is_path_allowed_for_scope(
|
||||
token_scope, method, path
|
||||
):
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
response = JSONResponse(
|
||||
{"detail": "Token à portée limitée : endpoint hors périmètre 'files'"},
|
||||
status_code=403,
|
||||
)
|
||||
await response(scope, receive, send)
|
||||
return
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def _scope_from_asgi(asgi_scope: Any) -> str | None:
|
||||
"""Extract the unverified ``scope`` claim from an ASGI scope's auth."""
|
||||
headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in asgi_scope.get("headers", [])}
|
||||
token: str | None = None
|
||||
auth = headers.get("authorization", "")
|
||||
if auth.lower().startswith("bearer "):
|
||||
token = auth[7:].strip()
|
||||
else:
|
||||
for part in headers.get("cookie", "").split(";"):
|
||||
name, _, value = part.strip().partition("=")
|
||||
if name.strip() == "access_token" and value.strip():
|
||||
token = value.strip()
|
||||
break
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
from jose import jwt as _jwt
|
||||
|
||||
claims = _jwt.get_unverified_claims(token)
|
||||
except Exception:
|
||||
return None
|
||||
token_scope = (claims or {}).get("scope")
|
||||
return token_scope if isinstance(token_scope, str) and token_scope else None
|
||||
+78
-53
@@ -6,8 +6,9 @@
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from .password import hash_password
|
||||
@@ -16,6 +17,12 @@ logger = logging.getLogger("obsigate.auth.users")
|
||||
|
||||
USERS_FILE = Path("data/users.json")
|
||||
|
||||
# Serialises read-modify-write cycles on users.json. ``RLock`` because a few
|
||||
# helpers (e.g. ``record_login_failure``) call other mutators while holding it.
|
||||
# BUG-029: without this, concurrent MFA enable + password change could lose one
|
||||
# of the two updates (last writer wins).
|
||||
_users_lock = threading.RLock()
|
||||
|
||||
|
||||
def _read() -> dict:
|
||||
"""Read users.json. Returns empty structure if file doesn't exist."""
|
||||
@@ -73,28 +80,40 @@ def create_user(
|
||||
role: str = "user",
|
||||
vaults: list[str] | None = None,
|
||||
display_name: str | None = None,
|
||||
must_change_password: bool = False,
|
||||
) -> dict:
|
||||
"""Create a new user. Raises ValueError if username already taken."""
|
||||
data = _read()
|
||||
if username in data["users"]:
|
||||
raise ValueError(f"User '{username}' already exists")
|
||||
"""Create a new user. Raises ValueError if username already taken.
|
||||
|
||||
user = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"username": username,
|
||||
"display_name": display_name or username,
|
||||
"password_hash": hash_password(password),
|
||||
"role": role,
|
||||
"vaults": vaults or [],
|
||||
"active": True,
|
||||
"language": "fr", # default UI language
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"last_login": None,
|
||||
"failed_attempts": 0,
|
||||
"locked_until": None,
|
||||
}
|
||||
data["users"][username] = user
|
||||
_write(data)
|
||||
``must_change_password`` (HUB-13) marque un mot de passe temporaire
|
||||
(création/réinitialisation par l'administration machine) : le
|
||||
changement est forcé à la première connexion. Les comptes créés
|
||||
avant ce champ n'en portent pas — ils valent ``False`` partout où
|
||||
le champ est lu via ``.get(..., False)``.
|
||||
"""
|
||||
with _users_lock:
|
||||
data = _read()
|
||||
if username in data["users"]:
|
||||
raise ValueError(f"User '{username}' already exists")
|
||||
|
||||
user = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"username": username,
|
||||
"display_name": display_name or username,
|
||||
"password_hash": hash_password(password),
|
||||
"role": role,
|
||||
"vaults": vaults or [],
|
||||
"active": True,
|
||||
"must_change_password": bool(must_change_password), # HUB-13
|
||||
"language": "fr", # default UI language
|
||||
"avatar": None, # profile picture data-URL (#113)
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"password_changed_at": datetime.now(UTC).timestamp(),
|
||||
"last_login": None,
|
||||
"failed_attempts": 0,
|
||||
"locked_until": None,
|
||||
}
|
||||
data["users"][username] = user
|
||||
_write(data)
|
||||
logger.info(f"Created user '{username}' (role={role})")
|
||||
return {k: v for k, v in user.items() if k != "password_hash"}
|
||||
|
||||
@@ -105,35 +124,40 @@ def update_user(username: str, updates: dict) -> dict:
|
||||
Forbidden fields (id, username, created_at) are silently ignored.
|
||||
If 'password' is in updates, it's hashed and stored as password_hash.
|
||||
"""
|
||||
data = _read()
|
||||
if username not in data["users"]:
|
||||
raise ValueError(f"User '{username}' not found")
|
||||
with _users_lock:
|
||||
data = _read()
|
||||
if username not in data["users"]:
|
||||
raise ValueError(f"User '{username}' not found")
|
||||
|
||||
forbidden = {"id", "username", "created_at"}
|
||||
safe_updates = {k: v for k, v in updates.items() if k not in forbidden}
|
||||
forbidden = {"id", "username", "created_at"}
|
||||
safe_updates = {k: v for k, v in updates.items() if k not in forbidden}
|
||||
|
||||
if "password" in safe_updates:
|
||||
safe_updates["password_hash"] = hash_password(safe_updates.pop("password"))
|
||||
if "password" in safe_updates:
|
||||
safe_updates["password_hash"] = hash_password(safe_updates.pop("password"))
|
||||
# BUG-028: invalidate every token issued before this change.
|
||||
safe_updates["password_changed_at"] = datetime.now(UTC).timestamp()
|
||||
|
||||
data["users"][username].update(safe_updates)
|
||||
_write(data)
|
||||
return {k: v for k, v in data["users"][username].items() if k != "password_hash"}
|
||||
data["users"][username].update(safe_updates)
|
||||
_write(data)
|
||||
result = {k: v for k, v in data["users"][username].items() if k != "password_hash"}
|
||||
return result
|
||||
|
||||
|
||||
def delete_user(username: str):
|
||||
"""Delete a user. Raises ValueError if not found."""
|
||||
data = _read()
|
||||
if username not in data["users"]:
|
||||
raise ValueError(f"User '{username}' not found")
|
||||
del data["users"][username]
|
||||
_write(data)
|
||||
with _users_lock:
|
||||
data = _read()
|
||||
if username not in data["users"]:
|
||||
raise ValueError(f"User '{username}' not found")
|
||||
del data["users"][username]
|
||||
_write(data)
|
||||
logger.info(f"Deleted user '{username}'")
|
||||
|
||||
|
||||
def record_login_success(username: str):
|
||||
"""Record a successful login: update last_login, reset failed attempts."""
|
||||
update_user(username, {
|
||||
"last_login": datetime.now(timezone.utc).isoformat(),
|
||||
"last_login": datetime.now(UTC).isoformat(),
|
||||
"failed_attempts": 0,
|
||||
"locked_until": None,
|
||||
})
|
||||
@@ -144,24 +168,25 @@ def record_login_failure(username: str) -> int:
|
||||
|
||||
After 5 failures, locks the account for 15 minutes.
|
||||
"""
|
||||
data = _read()
|
||||
user = data["users"].get(username)
|
||||
if not user:
|
||||
return 0
|
||||
with _users_lock:
|
||||
data = _read()
|
||||
user = data["users"].get(username)
|
||||
if not user:
|
||||
return 0
|
||||
|
||||
attempts = user.get("failed_attempts", 0) + 1
|
||||
updates = {"failed_attempts": attempts}
|
||||
attempts = user.get("failed_attempts", 0) + 1
|
||||
updates = {"failed_attempts": attempts}
|
||||
|
||||
# Lock after 5 failed attempts (15 minutes)
|
||||
if attempts >= 5:
|
||||
locked_until = (
|
||||
datetime.now(timezone.utc) + timedelta(minutes=15)
|
||||
).isoformat()
|
||||
updates["locked_until"] = locked_until
|
||||
logger.warning(f"Account '{username}' locked after {attempts} failed attempts")
|
||||
# Lock after 5 failed attempts (15 minutes)
|
||||
if attempts >= 5:
|
||||
locked_until = (
|
||||
datetime.now(UTC) + timedelta(minutes=15)
|
||||
).isoformat()
|
||||
updates["locked_until"] = locked_until
|
||||
logger.warning(f"Account '{username}' locked after {attempts} failed attempts")
|
||||
|
||||
update_user(username, updates)
|
||||
return attempts
|
||||
update_user(username, updates)
|
||||
return attempts
|
||||
|
||||
|
||||
def is_locked(username: str) -> bool:
|
||||
@@ -173,7 +198,7 @@ def is_locked(username: str) -> bool:
|
||||
if not user or not user.get("locked_until"):
|
||||
return False
|
||||
locked_until = datetime.fromisoformat(user["locked_until"])
|
||||
if datetime.now(timezone.utc) > locked_until:
|
||||
if datetime.now(UTC) > locked_until:
|
||||
# Lock expired — unlock
|
||||
update_user(username, {"locked_until": None, "failed_attempts": 0})
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
# backend/auth/webauthn_mfa.py
|
||||
# WebAuthn support for MFA (ROADMAP #64): security keys / platform biometrics.
|
||||
# Thin wrapper over the `webauthn` library with an in-memory challenge store.
|
||||
#
|
||||
# Credentials are persisted in users.json under "webauthn_credentials":
|
||||
# [{ "credential_id": <b64url>, "public_key": <b64url>, "sign_count": int,
|
||||
# "transports": [...], "label": str, "registered_at": iso }]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from webauthn import (
|
||||
generate_authentication_options,
|
||||
generate_registration_options,
|
||||
options_to_json,
|
||||
verify_authentication_response,
|
||||
verify_registration_response,
|
||||
)
|
||||
from webauthn.helpers import (
|
||||
base64url_to_bytes,
|
||||
bytes_to_base64url,
|
||||
parse_authentication_credential_json,
|
||||
parse_registration_credential_json,
|
||||
)
|
||||
from webauthn.helpers.structs import (
|
||||
AuthenticatorSelectionCriteria,
|
||||
ResidentKeyRequirement,
|
||||
UserVerificationRequirement,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.auth.webauthn")
|
||||
|
||||
# Challenge lifetime: clients have 3 minutes to complete the ceremony.
|
||||
CHALLENGE_TTL_SECONDS = 180
|
||||
|
||||
# How many outstanding challenges to keep per key. BUG-070: a single slot made
|
||||
# the flow fragile — a double-click on "add key" (or any retry) overwrote the
|
||||
# pending challenge and the in-flight ceremony failed with
|
||||
# "Client data challenge was not expected challenge". The verifier now accepts
|
||||
# any recent challenge for the key.
|
||||
MAX_PENDING_PER_KEY = 5
|
||||
|
||||
# In-memory pending challenges: key -> [(challenge_bytes, expires_at), ...]
|
||||
# (newest last)
|
||||
_pending: dict[str, list[tuple[bytes, float]]] = {}
|
||||
|
||||
|
||||
def rp_id() -> str:
|
||||
return os.environ.get("OBSIGATE_WEBAUTHN_RP_ID", "localhost")
|
||||
|
||||
|
||||
def rp_name() -> str:
|
||||
return os.environ.get("OBSIGATE_WEBAUTHN_RP_NAME", "ObsiGate")
|
||||
|
||||
|
||||
def expected_origins() -> list[str]:
|
||||
raw = os.environ.get("OBSIGATE_WEBAUTHN_ORIGINS", "http://localhost")
|
||||
return [o.strip() for o in raw.split(",") if o.strip()]
|
||||
|
||||
|
||||
def resolve_relying_party(request: Any = None) -> tuple[str, list[str]]:
|
||||
"""Resolve the WebAuthn (rp_id, expected_origins) for a ceremony.
|
||||
|
||||
BUG-070: the previous defaults (rp_id ``localhost``, origins
|
||||
``http://localhost``) rejected every real-world access URL — any port
|
||||
(``http://localhost:2020``), ``127.0.0.1``, a LAN host or a public domain
|
||||
failed verification with "Unexpected client data origin".
|
||||
|
||||
Explicit configuration still wins: when ``OBSIGATE_WEBAUTHN_RP_ID`` /
|
||||
``OBSIGATE_WEBAUTHN_ORIGINS`` are set they are used unchanged. Otherwise
|
||||
the values are derived from the incoming request (exact ``Host``, port
|
||||
included, since the browser origin carries non-default ports).
|
||||
|
||||
Behind a reverse proxy the external host/proto come from
|
||||
``X-Forwarded-Host`` / ``X-Forwarded-Proto``, honored only when
|
||||
``OBSIGATE_TRUST_PROXY=true`` (same rule as ``get_client_ip``).
|
||||
"""
|
||||
env_rp = os.environ.get("OBSIGATE_WEBAUTHN_RP_ID")
|
||||
env_raw = os.environ.get("OBSIGATE_WEBAUTHN_ORIGINS")
|
||||
if request is None:
|
||||
return (env_rp or "localhost",
|
||||
[o.strip() for o in env_raw.split(",") if o.strip()]
|
||||
if env_raw else ["http://localhost"])
|
||||
|
||||
from backend.services.net import is_trusted_proxy
|
||||
|
||||
if is_trusted_proxy():
|
||||
fwd_host = request.headers.get("x-forwarded-host", "")
|
||||
host = fwd_host.split(",")[0].strip() or request.headers.get("host", "")
|
||||
fwd_proto = request.headers.get("x-forwarded-proto", "")
|
||||
scheme = fwd_proto.split(",")[0].strip() or request.url.scheme
|
||||
else:
|
||||
host = request.headers.get("host", "")
|
||||
scheme = request.url.scheme
|
||||
if not host:
|
||||
url = request.url
|
||||
host = url.netloc or url.hostname or ""
|
||||
scheme = scheme or url.scheme or "http"
|
||||
rp = env_rp or _hostname_only(host) or "localhost"
|
||||
if env_raw:
|
||||
origins = [o.strip() for o in env_raw.split(",") if o.strip()]
|
||||
else:
|
||||
origins = [f"{scheme or 'http'}://{host}"] if host else ["http://localhost"]
|
||||
return rp, origins
|
||||
|
||||
|
||||
def _hostname_only(host: str) -> str:
|
||||
"""Strip the port (and IPv6 brackets) from a Host header value."""
|
||||
host = host.strip()
|
||||
if host.startswith("["): # [::1]:8080 or [::1]
|
||||
end = host.find("]")
|
||||
return host[1:end] if end > 0 else host
|
||||
if host.count(":") == 1:
|
||||
name, _, port = host.partition(":")
|
||||
return name if port.isdigit() else host
|
||||
return host
|
||||
|
||||
|
||||
def _prune_expired() -> None:
|
||||
now = time.time()
|
||||
for key in list(_pending):
|
||||
remaining = [(c, exp) for c, exp in _pending[key] if exp >= now]
|
||||
if remaining:
|
||||
_pending[key] = remaining
|
||||
else:
|
||||
_pending.pop(key, None)
|
||||
|
||||
|
||||
def _store_challenge(key: str) -> bytes:
|
||||
_prune_expired()
|
||||
challenge = secrets.token_bytes(32)
|
||||
slot = _pending.setdefault(key, [])
|
||||
slot.append((challenge, time.time() + CHALLENGE_TTL_SECONDS))
|
||||
del slot[:-MAX_PENDING_PER_KEY] # keep only the most recent ones
|
||||
return challenge
|
||||
|
||||
|
||||
def _take_challenge(key: str) -> bytes | None:
|
||||
"""Pop the newest challenge (single-use). Returns None if missing/expired."""
|
||||
_prune_expired()
|
||||
slot = _pending.get(key)
|
||||
if not slot:
|
||||
return None
|
||||
challenge, _ = slot.pop()
|
||||
if not slot:
|
||||
_pending.pop(key, None)
|
||||
return challenge
|
||||
|
||||
|
||||
def _take_all_challenges(key: str) -> list[bytes]:
|
||||
"""Pop every outstanding challenge for *key* (newest last)."""
|
||||
_prune_expired()
|
||||
slot = _pending.pop(key, None)
|
||||
return [c for c, _ in slot] if slot else []
|
||||
|
||||
|
||||
def clear_pending(username: str) -> None:
|
||||
"""Drop all pending challenges for a user (e.g. after enable/disable)."""
|
||||
for key in [k for k in _pending if k.startswith(f"{username}:")]:
|
||||
_pending.pop(key, None)
|
||||
|
||||
|
||||
# ── Registration (enrol a key in settings) ─────────────────────────────
|
||||
|
||||
def begin_registration(username: str, display_name: str,
|
||||
rp_id_override: str | None = None,
|
||||
origins_override: list[str] | None = None) -> dict:
|
||||
_ = origins_override # origins only matter at verification time
|
||||
options = generate_registration_options(
|
||||
rp_id=rp_id_override or rp_id(),
|
||||
rp_name=rp_name(),
|
||||
user_name=username,
|
||||
user_display_name=display_name or username,
|
||||
challenge=_store_challenge(f"{username}:register"),
|
||||
authenticator_selection=AuthenticatorSelectionCriteria(
|
||||
resident_key=ResidentKeyRequirement.PREFERRED,
|
||||
user_verification=UserVerificationRequirement.PREFERRED,
|
||||
),
|
||||
)
|
||||
return _finalize_options(options)
|
||||
|
||||
|
||||
def _verify_with_any_challenge(key: str, verify_one: Any, empty_message: str) -> Any:
|
||||
"""Run *verify_one(challenge)* against every outstanding challenge.
|
||||
|
||||
Returns the first success; re-raises the last error when all fail.
|
||||
BUG-070: lets an in-flight ceremony survive a re-requested options call
|
||||
(double-click / retry) that stored a newer challenge afterwards.
|
||||
"""
|
||||
challenges = _take_all_challenges(key)
|
||||
if not challenges:
|
||||
raise ValueError(empty_message)
|
||||
last_error: Exception | None = None
|
||||
for challenge in challenges:
|
||||
try:
|
||||
return verify_one(challenge)
|
||||
except Exception as e: # try the next candidate challenge
|
||||
last_error = e
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
|
||||
def complete_registration(username: str, credential_json: dict[str, Any],
|
||||
label: str = "", rp_id_override: str | None = None,
|
||||
origins_override: list[str] | None = None) -> dict:
|
||||
credential = parse_registration_credential_json(credential_json)
|
||||
effective_rp = rp_id_override or rp_id()
|
||||
effective_origins = origins_override or expected_origins()
|
||||
|
||||
def _verify(challenge: bytes) -> Any:
|
||||
return verify_registration_response(
|
||||
credential=credential,
|
||||
expected_challenge=challenge,
|
||||
expected_rp_id=effective_rp,
|
||||
expected_origin=effective_origins,
|
||||
)
|
||||
|
||||
verification = _verify_with_any_challenge(
|
||||
f"{username}:register", _verify,
|
||||
"Session d'enregistrement expirée — recommencez")
|
||||
|
||||
transports = credential.response.transports or []
|
||||
label = (label or str(credential_json.get("label") or "")).strip() or "Security key"
|
||||
record = {
|
||||
"credential_id": bytes_to_base64url(verification.credential_id),
|
||||
"public_key": bytes_to_base64url(verification.credential_public_key),
|
||||
"sign_count": int(verification.sign_count),
|
||||
"transports": [str(t) for t in transports],
|
||||
"label": label[:60],
|
||||
}
|
||||
return record
|
||||
|
||||
|
||||
# ── Authentication (assertion at login) ────────────────────────────────
|
||||
|
||||
def begin_authentication(username: str, credentials: list[dict],
|
||||
rp_id_override: str | None = None,
|
||||
origins_override: list[str] | None = None) -> dict | None:
|
||||
if not credentials:
|
||||
return None
|
||||
_ = origins_override # origins only matter at verification time
|
||||
from webauthn.helpers.structs import PublicKeyCredentialDescriptor
|
||||
|
||||
allow = [
|
||||
PublicKeyCredentialDescriptor(id=base64url_to_bytes(c["credential_id"]))
|
||||
for c in credentials
|
||||
]
|
||||
options = generate_authentication_options(
|
||||
rp_id=rp_id_override or rp_id(),
|
||||
challenge=_store_challenge(f"{username}:login"),
|
||||
allow_credentials=allow,
|
||||
)
|
||||
return _finalize_options(options)
|
||||
|
||||
|
||||
def complete_authentication(
|
||||
username: str,
|
||||
credential_json: dict[str, Any],
|
||||
stored: dict,
|
||||
rp_id_override: str | None = None,
|
||||
origins_override: list[str] | None = None,
|
||||
) -> int:
|
||||
"""Verify an assertion. Returns the new sign_count. Raises on failure."""
|
||||
credential = parse_authentication_credential_json(credential_json)
|
||||
effective_rp = rp_id_override or rp_id()
|
||||
effective_origins = origins_override or expected_origins()
|
||||
|
||||
def _verify(challenge: bytes) -> Any:
|
||||
return verify_authentication_response(
|
||||
credential=credential,
|
||||
expected_challenge=challenge,
|
||||
expected_rp_id=effective_rp,
|
||||
expected_origin=effective_origins,
|
||||
credential_public_key=base64url_to_bytes(stored["public_key"]),
|
||||
credential_current_sign_count=int(stored.get("sign_count", 0)),
|
||||
)
|
||||
|
||||
verification = _verify_with_any_challenge(
|
||||
f"{username}:login", _verify,
|
||||
"Session expirée — rechargez la page")
|
||||
return int(verification.new_sign_count)
|
||||
|
||||
|
||||
def credentials_for_api(credentials: list[dict]) -> list[dict]:
|
||||
"""Sanitized view for the settings UI (no public keys)."""
|
||||
return [
|
||||
{
|
||||
"credential_id": c.get("credential_id"),
|
||||
"label": c.get("label", "Security key"),
|
||||
"transports": c.get("transports", []),
|
||||
"registered_at": c.get("registered_at"),
|
||||
}
|
||||
for c in credentials
|
||||
]
|
||||
|
||||
|
||||
def _finalize_options(options) -> dict:
|
||||
import json
|
||||
|
||||
return json.loads(options_to_json(options))
|
||||
@@ -0,0 +1,662 @@
|
||||
"""BooksLM — Context collection and caching for directory-scoped AI chat.
|
||||
|
||||
Collects markdown files from an Obsidian vault directory, applies secret
|
||||
redaction, builds a system prompt with file contents, and caches results
|
||||
for repeated queries.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.media_types import is_media
|
||||
from backend.secret_redactor import redact_file_content
|
||||
|
||||
logger = logging.getLogger("obsigate.bookslm")
|
||||
|
||||
# ── Configuration limits ──
|
||||
BOOKSLM_MAX_FILES = int(os.getenv("BOOKSLM_MAX_FILES", "200"))
|
||||
BOOKSLM_MAX_TOTAL_CHARS = int(os.getenv("BOOKSLM_MAX_TOTAL_CHARS", "200000"))
|
||||
BOOKSLM_MAX_FILE_CHARS = int(os.getenv("BOOKSLM_MAX_FILE_CHARS", "30000"))
|
||||
# Maximum size of an image sent to a vision model (bytes, before base64).
|
||||
BOOKSLM_MAX_IMAGE_BYTES = int(os.getenv("BOOKSLM_MAX_IMAGE_BYTES", "10000000"))
|
||||
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".avif"}
|
||||
|
||||
# ── Cache ──
|
||||
_cache: dict[str, dict[str, Any]] = {}
|
||||
_CACHE_TTL = 300 # seconds
|
||||
|
||||
|
||||
def _cache_key(vault_path: Path, directory: str, file_mtimes: list[tuple[str, float]]) -> str:
|
||||
"""Build a SHA-256 cache key from vault+directory+file modification times."""
|
||||
raw = json.dumps({
|
||||
"vault": str(vault_path),
|
||||
"dir": directory,
|
||||
"mtimes": sorted(file_mtimes),
|
||||
}, sort_keys=True)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()
|
||||
|
||||
|
||||
def _should_skip(name: str) -> bool:
|
||||
"""Return True if this file/directory name should be skipped."""
|
||||
skip_prefixes = (".",)
|
||||
skip_names = {"_attachments", "node_modules", ".git", ".obsidian", "__pycache__"}
|
||||
if name in skip_names:
|
||||
return True
|
||||
return bool(any(name.startswith(p) for p in skip_prefixes))
|
||||
|
||||
|
||||
def _file_priority(path: Path) -> tuple[int, float]:
|
||||
"""Sort key: README/index first, then by modification time descending.
|
||||
|
||||
Returns (priority_group, -mtime) so that:
|
||||
- Group 0: README* and index* files (come first)
|
||||
- Group 1: all other files (come after)
|
||||
Within each group, newer files come first.
|
||||
"""
|
||||
name_lower = path.stem.lower()
|
||||
if name_lower.startswith("readme") or name_lower.startswith("index"):
|
||||
group = 0
|
||||
else:
|
||||
group = 1
|
||||
try:
|
||||
mtime = path.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
return (group, -mtime)
|
||||
|
||||
|
||||
def collect_directory_context(vault_path: Path, directory: str) -> dict[str, Any]:
|
||||
"""Walk a directory recursively, collect .md files with content.
|
||||
|
||||
Args:
|
||||
vault_path: Absolute path to the vault root.
|
||||
directory: Relative directory path within the vault (empty = root).
|
||||
|
||||
Returns:
|
||||
Dict with keys: files, total_chars, file_count, directory_tree.
|
||||
"""
|
||||
target_dir = (vault_path / directory).resolve() if directory else vault_path.resolve()
|
||||
vault_resolved = vault_path.resolve()
|
||||
|
||||
# Safety: ensure target is within vault
|
||||
try:
|
||||
target_dir.relative_to(vault_resolved)
|
||||
except ValueError:
|
||||
logger.warning(f"Directory outside vault: {target_dir}")
|
||||
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
|
||||
|
||||
if not target_dir.exists() or not target_dir.is_dir():
|
||||
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
|
||||
|
||||
# Check cache
|
||||
file_mtimes: list[tuple[str, float]] = []
|
||||
md_files: list[Path] = []
|
||||
try:
|
||||
for p in target_dir.rglob("*"):
|
||||
# Skip hidden dirs/files and special dirs
|
||||
parts = p.relative_to(target_dir).parts
|
||||
if any(_should_skip(part) for part in parts):
|
||||
continue
|
||||
if p.is_file() and p.suffix.lower() == ".md":
|
||||
md_files.append(p)
|
||||
try:
|
||||
file_mtimes.append((str(p.relative_to(target_dir)), p.stat().st_mtime))
|
||||
except OSError:
|
||||
file_mtimes.append((str(p.relative_to(target_dir)), 0.0))
|
||||
except PermissionError:
|
||||
logger.warning(f"Permission denied scanning {target_dir}")
|
||||
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
|
||||
|
||||
key = _cache_key(vault_resolved, directory, file_mtimes)
|
||||
if key in _cache:
|
||||
cached = _cache[key]
|
||||
if time.time() - cached.get("_ts", 0) < _CACHE_TTL:
|
||||
logger.debug(f"Cache hit for {directory}")
|
||||
return {k: v for k, v in cached.items() if k != "_ts"}
|
||||
|
||||
# Sort by priority: README/index first, then by mtime descending
|
||||
md_files.sort(key=_file_priority)
|
||||
|
||||
# Apply limits
|
||||
collected: list[dict[str, Any]] = []
|
||||
total_chars = 0
|
||||
|
||||
for p in md_files:
|
||||
if len(collected) >= BOOKSLM_MAX_FILES:
|
||||
break
|
||||
if total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
|
||||
break
|
||||
|
||||
rel_path = str(p.relative_to(vault_resolved)).replace("\\", "/")
|
||||
try:
|
||||
content = p.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
logger.warning(f"Cannot read {rel_path}: {e}")
|
||||
continue
|
||||
|
||||
# Redact secrets
|
||||
content = redact_file_content(content, rel_path)
|
||||
|
||||
# Truncate if too long
|
||||
if len(content) > BOOKSLM_MAX_FILE_CHARS:
|
||||
content = content[:BOOKSLM_MAX_FILE_CHARS] + "\n\n[... tronqué]"
|
||||
|
||||
remaining = BOOKSLM_MAX_TOTAL_CHARS - total_chars
|
||||
if len(content) > remaining:
|
||||
content = content[:remaining] + "\n\n[... tronqué]"
|
||||
|
||||
title = p.stem.replace("-", " ").replace("_", " ").title()
|
||||
file_type = "markdown"
|
||||
|
||||
collected.append({
|
||||
"path": rel_path,
|
||||
"title": title,
|
||||
"content": content,
|
||||
"type": file_type,
|
||||
})
|
||||
total_chars += len(content)
|
||||
|
||||
# Build directory tree
|
||||
dir_tree = _build_directory_tree(target_dir, vault_resolved)
|
||||
|
||||
result = {
|
||||
"files": collected,
|
||||
"total_chars": total_chars,
|
||||
"file_count": len(collected),
|
||||
"directory_tree": dir_tree,
|
||||
"max_total_chars": BOOKSLM_MAX_TOTAL_CHARS,
|
||||
"max_files": BOOKSLM_MAX_FILES,
|
||||
"scope": "directory",
|
||||
}
|
||||
|
||||
# Store in cache
|
||||
_cache[key] = {**result, "_ts": time.time()}
|
||||
logger.info(f"Collected {len(collected)} files ({total_chars} chars) from {directory or '/'}")
|
||||
return result
|
||||
|
||||
|
||||
def _file_entry(target: Path, rel_path: str, remaining: int) -> dict[str, Any] | None:
|
||||
"""Read, redact and truncate a single file into a context entry."""
|
||||
suffix = target.suffix.lower()
|
||||
# #109-D3 — audio/video (and images) carry no extractable text; never feed
|
||||
# raw bytes to the model. Images are handled separately via vision data URLs.
|
||||
if is_media(suffix):
|
||||
return None
|
||||
try:
|
||||
if suffix == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_text
|
||||
|
||||
content = extract_pdf_text(target)
|
||||
file_type = "pdf"
|
||||
else:
|
||||
content = target.read_text(encoding="utf-8", errors="replace")
|
||||
file_type = "markdown"
|
||||
except Exception as e:
|
||||
logger.warning(f"Cannot read {rel_path}: {e}")
|
||||
return None
|
||||
|
||||
content = redact_file_content(content, rel_path)
|
||||
if len(content) > BOOKSLM_MAX_FILE_CHARS:
|
||||
content = content[:BOOKSLM_MAX_FILE_CHARS] + "\n\n[... tronqué]"
|
||||
if len(content) > remaining:
|
||||
content = content[:remaining] + "\n\n[... tronqué]"
|
||||
|
||||
return {
|
||||
"path": rel_path,
|
||||
"title": target.stem.replace("-", " ").replace("_", " ").title(),
|
||||
"content": content,
|
||||
"type": file_type,
|
||||
}
|
||||
|
||||
|
||||
def collect_files_context(
|
||||
vault_path: Path,
|
||||
rel_paths: list[str],
|
||||
scope: str = "documents",
|
||||
) -> dict[str, Any]:
|
||||
"""Collect an explicit list of files (open documents) as AI context.
|
||||
|
||||
Unlike :func:`collect_directory_context`, this reads only the requested
|
||||
relative paths (markdown or PDF), never the whole directory. Paths outside
|
||||
the vault are silently ignored.
|
||||
|
||||
Args:
|
||||
vault_path: Absolute path to the vault root.
|
||||
rel_paths: Relative file paths within the vault (in display order).
|
||||
scope: Context label exposed to the UI ("documents").
|
||||
|
||||
Returns:
|
||||
Same shape as :func:`collect_directory_context`.
|
||||
"""
|
||||
vault_resolved = vault_path.resolve()
|
||||
collected: list[dict[str, Any]] = []
|
||||
total_chars = 0
|
||||
seen: set[str] = set()
|
||||
|
||||
for rel in rel_paths:
|
||||
if len(collected) >= BOOKSLM_MAX_FILES or total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
|
||||
break
|
||||
if not rel or rel in seen:
|
||||
continue
|
||||
seen.add(rel)
|
||||
try:
|
||||
target = (vault_resolved / rel).resolve()
|
||||
target.relative_to(vault_resolved)
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
if not target.is_file():
|
||||
continue
|
||||
entry = _file_entry(target, rel, BOOKSLM_MAX_TOTAL_CHARS - total_chars)
|
||||
if entry is None:
|
||||
continue
|
||||
collected.append(entry)
|
||||
total_chars += len(entry["content"])
|
||||
|
||||
return {
|
||||
"files": collected,
|
||||
"total_chars": total_chars,
|
||||
"file_count": len(collected),
|
||||
"directory_tree": "",
|
||||
"max_total_chars": BOOKSLM_MAX_TOTAL_CHARS,
|
||||
"max_files": BOOKSLM_MAX_FILES,
|
||||
"scope": scope,
|
||||
}
|
||||
|
||||
|
||||
def collect_adhoc_context(
|
||||
vault_path: Path,
|
||||
files: list[str] | None = None,
|
||||
directories: list[str] | None = None,
|
||||
scope: str = "general",
|
||||
) -> dict[str, Any]:
|
||||
"""Collect an ad-hoc mix of explicit files and directories.
|
||||
|
||||
Backs the assistant ``@`` command: the user can attach extra files and
|
||||
directories to the current context without changing the base mode. Paths
|
||||
outside the vault are ignored by the underlying collectors.
|
||||
|
||||
Args:
|
||||
vault_path: Absolute path to the vault root.
|
||||
files: Relative file paths to include.
|
||||
directories: Relative directory paths to include (recursive).
|
||||
scope: Context label exposed to the UI.
|
||||
|
||||
Returns:
|
||||
Same shape as :func:`collect_directory_context`.
|
||||
"""
|
||||
vault_resolved = vault_path.resolve()
|
||||
collected: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
total_chars = 0
|
||||
|
||||
def _add(entries: list[dict[str, Any]]) -> None:
|
||||
nonlocal total_chars
|
||||
for entry in entries:
|
||||
if len(collected) >= BOOKSLM_MAX_FILES or total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
|
||||
return
|
||||
path = entry.get("path")
|
||||
if not path or path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
collected.append(entry)
|
||||
total_chars += len(entry.get("content", ""))
|
||||
|
||||
if files:
|
||||
_add(collect_files_context(vault_resolved, files, scope=scope)["files"])
|
||||
for directory in directories or []:
|
||||
_add(collect_directory_context(vault_resolved, directory)["files"])
|
||||
|
||||
return {
|
||||
"files": collected,
|
||||
"total_chars": total_chars,
|
||||
"file_count": len(collected),
|
||||
"directory_tree": "",
|
||||
"max_total_chars": BOOKSLM_MAX_TOTAL_CHARS,
|
||||
"max_files": BOOKSLM_MAX_FILES,
|
||||
"scope": scope,
|
||||
}
|
||||
|
||||
|
||||
def merge_contexts(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Merge two context payloads, de-duplicating files by path."""
|
||||
files: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
total_chars = 0
|
||||
for entry in list(base.get("files", [])) + list(extra.get("files", [])):
|
||||
path = entry.get("path")
|
||||
if not path or path in seen:
|
||||
continue
|
||||
if len(files) >= BOOKSLM_MAX_FILES or total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
|
||||
break
|
||||
seen.add(path)
|
||||
files.append(entry)
|
||||
total_chars += len(entry.get("content", ""))
|
||||
|
||||
tree = base.get("directory_tree", "") or extra.get("directory_tree", "")
|
||||
scope = extra.get("scope") or base.get("scope", "general")
|
||||
return {
|
||||
"files": files,
|
||||
"total_chars": total_chars,
|
||||
"file_count": len(files),
|
||||
"directory_tree": tree,
|
||||
"max_total_chars": BOOKSLM_MAX_TOTAL_CHARS,
|
||||
"max_files": BOOKSLM_MAX_FILES,
|
||||
"scope": scope,
|
||||
}
|
||||
|
||||
|
||||
def is_image_path(path: str) -> bool:
|
||||
"""True when the path has a supported image extension."""
|
||||
return Path(path or "").suffix.lower() in IMAGE_EXTENSIONS
|
||||
|
||||
|
||||
def load_vault_image_data_url(vault_path: Path, rel_path: str) -> str | None:
|
||||
"""Read a vault image and return it as a ``data:`` URL for vision models.
|
||||
|
||||
Returns ``None`` when the path is outside the vault, missing, too large,
|
||||
or not a supported image.
|
||||
"""
|
||||
if not rel_path or not is_image_path(rel_path):
|
||||
return None
|
||||
vault_resolved = vault_path.resolve()
|
||||
try:
|
||||
target = (vault_resolved / rel_path).resolve()
|
||||
target.relative_to(vault_resolved)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
if not target.is_file():
|
||||
return None
|
||||
try:
|
||||
if target.stat().st_size > BOOKSLM_MAX_IMAGE_BYTES:
|
||||
logger.warning("Image too large to send: %s", rel_path)
|
||||
return None
|
||||
raw = target.read_bytes()
|
||||
except OSError as exc:
|
||||
logger.warning("Cannot read image %s: %s", rel_path, exc)
|
||||
return None
|
||||
mime = mimetypes.guess_type(str(target))[0] or "image/png"
|
||||
encoded = base64.b64encode(raw).decode("ascii")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
def empty_context(scope: str = "general") -> dict[str, Any]:
|
||||
"""Return an empty context payload (used by the General assistant)."""
|
||||
return {
|
||||
"files": [],
|
||||
"total_chars": 0,
|
||||
"file_count": 0,
|
||||
"directory_tree": "",
|
||||
"max_total_chars": BOOKSLM_MAX_TOTAL_CHARS,
|
||||
"max_files": BOOKSLM_MAX_FILES,
|
||||
"scope": scope,
|
||||
}
|
||||
|
||||
|
||||
def _build_directory_tree(target_dir: Path, vault_root: Path) -> str:
|
||||
"""Build a text representation of the directory tree (dirs + .md files)."""
|
||||
lines: list[str] = []
|
||||
try:
|
||||
for p in sorted(target_dir.rglob("*")):
|
||||
parts = p.relative_to(target_dir).parts
|
||||
if any(_should_skip(part) for part in parts):
|
||||
continue
|
||||
if p.is_dir():
|
||||
depth = len(p.relative_to(target_dir).parts)
|
||||
lines.append(f"{' ' * depth}{p.name}/")
|
||||
elif p.is_file() and p.suffix.lower() == ".md":
|
||||
depth = len(p.relative_to(target_dir).parts)
|
||||
lines.append(f"{' ' * depth}{p.name}")
|
||||
except PermissionError:
|
||||
pass
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_system_prompt(context: dict[str, Any], scope: str = "directory", vault_name: str | None = None) -> str:
|
||||
"""Build a system prompt for document-scoped AI chat.
|
||||
|
||||
Args:
|
||||
context: Output of collect_directory_context()/collect_files_context().
|
||||
scope: "directory" (whole folder) or "documents" (open files).
|
||||
vault_name: Vault the context files belong to. When set, the prompt
|
||||
states it explicitly with write-tool guidance (BUG-046: without
|
||||
it the model invented vault names — e.g. "test" — and every
|
||||
confirmed ``append_to_file``/``edit_file`` call failed).
|
||||
|
||||
Returns:
|
||||
System prompt string with file contents.
|
||||
"""
|
||||
files = context.get("files", [])
|
||||
file_count = context.get("file_count", 0)
|
||||
total_chars = context.get("total_chars", 0)
|
||||
|
||||
# Rough token estimate (1 token ≈ 4 chars)
|
||||
est_tokens = total_chars // 4
|
||||
token_warning = ""
|
||||
if est_tokens > 100_000:
|
||||
token_warning = f"\n⚠️ Attention : le contexte est très volumineux (~{est_tokens:,} tokens estimés). Les réponses peuvent être moins précises.\n"
|
||||
|
||||
if scope == "documents":
|
||||
role = (
|
||||
"Tu es un assistant de recherche documentaire intégré à ObsiGate. "
|
||||
"Tu réponds UNIQUEMENT en te basant sur les documents ouverts par l'utilisateur ci-dessous. "
|
||||
"Cite tes sources avec le nom du fichier quand tu utilises une information. "
|
||||
"Si l'information ne se trouve pas dans les documents, dis-le clairement."
|
||||
)
|
||||
context_label = "📄 Documents ouverts"
|
||||
else:
|
||||
role = (
|
||||
"Tu es un assistant de recherche documentaire. "
|
||||
"Tu réponds UNIQUEMENT en te basant sur les documents fournis ci-dessous. "
|
||||
"Cite tes sources avec le nom du fichier quand tu utilises une information. "
|
||||
"Si l'information ne se trouve pas dans les documents, dis-le clairement."
|
||||
)
|
||||
context_label = "📚 Contexte"
|
||||
|
||||
prompt = (
|
||||
f"{role}"
|
||||
f"\n\n{context_label} : {file_count} fichier(s) ({total_chars:,} caractères)"
|
||||
f"{token_warning}\n"
|
||||
)
|
||||
|
||||
# Directory tree
|
||||
tree = context.get("directory_tree", "")
|
||||
if tree:
|
||||
prompt += f"\n📂 Arborescence du dossier :\n```\n{tree}\n```\n"
|
||||
|
||||
# File contents
|
||||
prompt += "\n---\n"
|
||||
for f in files:
|
||||
prompt += f"\n## 📄 {f['title']} (`{f['path']}`)\n\n{f['content']}\n\n---\n"
|
||||
|
||||
prompt += "\nFin du contexte. Réponds à la question de l'utilisateur en te basant uniquement sur ces documents."
|
||||
|
||||
if vault_name:
|
||||
prompt += (
|
||||
f"\n\nCes documents appartiennent au vault « {vault_name} ». Quand tu utilises un outil "
|
||||
"d'écriture (`append_to_file`, `edit_file`, `create_file`), passe TOUJOURS "
|
||||
f"exactement `\"vault\": \"{vault_name}\"` (jamais un nom inventé) et un `path` "
|
||||
"relatif au vault, identique à celui affiché ci-dessus. Pour créer un fichier "
|
||||
"dans un nouveau dossier, un seul `create_file` avec le chemin complet suffit "
|
||||
"(les dossiers parents sont créés automatiquement)."
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
GENERAL_SYSTEM_HEADER = """Tu es l'assistant intégré d'ObsiGate, une application web auto-hébergée pour consulter, rechercher et éditer des vaults Obsidian (Markdown).
|
||||
|
||||
Tes deux rôles :
|
||||
1. **Aider sur l'application** : expliquer la navigation, la recherche (full-text, filtres `tag:`, `created:`, `path:`), l'éditeur (CodeMirror, autosave, raccourcis), les onglets et le split view, les sauvegardes et la restauration, le partage public, l'export (HTML/Markdown/ePub/PDF), Mermaid, Excalidraw, les plugins, les thèmes, le mode hors-ligne, le MFA, etc.
|
||||
2. **Proposer des actions concrètes** : créer un fichier ou un dossier dans un vault.
|
||||
"""
|
||||
|
||||
# Text action protocol — used by the classic (non-agent) chat endpoint, where
|
||||
# the model has no native tool calling; the frontend turns each block into a
|
||||
# clickable “Apply” card.
|
||||
GENERAL_ACTION_TEXT_PROTOCOL = """
|
||||
Quand l'utilisateur demande explicitement de créer un fichier, inclus un bloc de ce type dans ta réponse (un bloc par fichier, et rien d'autre à l'intérieur du bloc) :
|
||||
|
||||
```obsigate-action
|
||||
{"action": "create_file", "vault": "<nom du vault>", "path": "<chemin/relatif.md>", "content": "<contenu markdown>"}
|
||||
```
|
||||
|
||||
Pour créer un dossier :
|
||||
|
||||
```obsigate-action
|
||||
{"action": "create_directory", "vault": "<nom du vault>", "path": "<chemin/relatif>"}
|
||||
```
|
||||
|
||||
Règles :
|
||||
- Ne propose une action que si l'utilisateur la demande explicitement.
|
||||
- Explique en une phrase ce que fait l'action avant le bloc.
|
||||
- Utilise un chemin relatif se terminant par `.md` pour un fichier.
|
||||
- Pour créer un fichier dans un nouveau dossier, utilise **un seul** bloc `create_file` avec le chemin complet (ex. `"path": "Dossier/fichier.md"`) : les dossiers parents sont créés automatiquement, inutile d'émettre un `create_directory` séparé.
|
||||
- N'invente jamais un nom de vault : utilise l'un des vaults disponibles listés ci-dessous.
|
||||
- Réponds dans la langue de l'utilisateur, de façon concise et structurée (Markdown).
|
||||
"""
|
||||
|
||||
# Agent mode: the model has native tools, so it must call them (function
|
||||
# calling) instead of emitting the text `obsigate-action` blocks — otherwise
|
||||
# the requested file is never created (BUG-053).
|
||||
GENERAL_ACTION_TOOL_PROTOCOL = """
|
||||
Tu disposes d'outils natifs (function calling) pour lire, chercher et modifier les vaults : `create_file`, `create_directory`, `append_to_file`, `edit_file`, `read_file`, `search_fulltext`, etc.
|
||||
|
||||
Quand l'utilisateur demande explicitement de créer un fichier, **appelle directement l'outil `create_file`** avec `{"vault": "<nom du vault>", "path": "<chemin/relatif.md>", "content": "<contenu markdown>"}`. Pour créer un dossier, appelle `create_directory`.
|
||||
|
||||
Règles :
|
||||
- N'écris **jamais** de bloc ```obsigate-action``` : en mode agent, toutes les actions passent par les outils natifs.
|
||||
- Écris le contenu **complet** demandé dans l'argument `content` (ne le tronque pas, pas de « … » ni de ligne omise).
|
||||
- Pour créer un fichier dans un nouveau dossier, un seul appel `create_file` avec le chemin complet suffit (les dossiers parents sont créés automatiquement).
|
||||
- N'invente jamais un nom de vault : utilise l'un des vaults disponibles listés ci-dessous.
|
||||
- Réponds dans la langue de l'utilisateur, de façon concise et structurée (Markdown).
|
||||
"""
|
||||
|
||||
# Backwards-compatible alias (classic chat prompt).
|
||||
GENERAL_SYSTEM_PROMPT = GENERAL_SYSTEM_HEADER + GENERAL_ACTION_TEXT_PROTOCOL
|
||||
|
||||
|
||||
def _format_app_context(app_context: dict[str, Any] | None, recent_files: list[dict[str, Any]] | None) -> str:
|
||||
"""Render the live application state for the General assistant prompt.
|
||||
|
||||
The General assistant has no document context; without this block it only
|
||||
knows the app exists. Passing what the user currently sees (open documents,
|
||||
current directory, active search, recently modified files) lets it answer
|
||||
"résume ce que je fais / où j'en suis" style questions.
|
||||
"""
|
||||
app_context = app_context or {}
|
||||
lines: list[str] = []
|
||||
|
||||
vault = app_context.get("vault") or app_context.get("current_vault")
|
||||
if vault:
|
||||
lines.append(f"- Vault sélectionné : {vault}")
|
||||
directory = app_context.get("directory")
|
||||
if directory:
|
||||
lines.append(f"- Répertoire courant : {directory}")
|
||||
current_path = app_context.get("current_path")
|
||||
if current_path:
|
||||
lines.append(f"- Document affiché dans le viewer : {current_path}")
|
||||
|
||||
docs = app_context.get("open_documents") or []
|
||||
rendered_docs = []
|
||||
for doc in docs:
|
||||
if not isinstance(doc, dict) or not doc.get("path"):
|
||||
continue
|
||||
rendered_docs.append(f"{doc['path']} (vault {doc['vault']})" if doc.get("vault") else str(doc["path"]))
|
||||
if rendered_docs:
|
||||
lines.append("- Documents ouverts dans les onglets/panneaux : " + ", ".join(rendered_docs))
|
||||
|
||||
editing = app_context.get("editing")
|
||||
if isinstance(editing, dict) and editing.get("path"):
|
||||
surface = "Forge" if editing.get("surface") == "forge" else "l'éditeur"
|
||||
location = f"{editing['path']} (vault {editing['vault']})" if editing.get("vault") else str(editing["path"])
|
||||
lines.append(
|
||||
f"- Document en cours d'édition dans {surface} : {location} — c'est le document affiché à la "
|
||||
"place de la vue lecture. Pour le mettre à jour, utilise les outils d'écriture "
|
||||
"(`edit_file`, `append_to_file`) : la modification est rechargée automatiquement dans "
|
||||
"l'éditeur et la vue lecture dès l'exécution de l'outil."
|
||||
)
|
||||
|
||||
query = app_context.get("search_query")
|
||||
if query:
|
||||
total = app_context.get("search_total")
|
||||
suffix = f" ({total} résultat(s))" if isinstance(total, int) else ""
|
||||
lines.append(f"- Recherche en cours : « {query} »{suffix}")
|
||||
|
||||
results = app_context.get("search_results") or []
|
||||
rendered_results = [str(r.get("path")) for r in results[:10] if isinstance(r, dict) and r.get("path")]
|
||||
if rendered_results:
|
||||
lines.append("- Premiers résultats affichés : " + ", ".join(rendered_results))
|
||||
|
||||
if recent_files:
|
||||
rendered_recent = [
|
||||
f"{f.get('vault')}/{f.get('path')}" for f in recent_files[:10] if f.get("path")
|
||||
]
|
||||
if rendered_recent:
|
||||
lines.append("- Fichiers récemment modifiés : " + ", ".join(rendered_recent))
|
||||
|
||||
if not lines:
|
||||
return ""
|
||||
return (
|
||||
"## Contexte applicatif actuel\n"
|
||||
"Voici ce que l'utilisateur voit ou fait en ce moment dans ObsiGate. "
|
||||
"Sers-t'en pour comprendre sa demande ; ne le répète pas inutilement.\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def build_general_system_prompt(
|
||||
vaults: list[str] | None = None,
|
||||
app_context: dict[str, Any] | None = None,
|
||||
recent_files: list[dict[str, Any]] | None = None,
|
||||
agent: bool = False,
|
||||
) -> str:
|
||||
"""System prompt for the General assistant (app help + actions).
|
||||
|
||||
``app_context`` carries the live UI state (open documents, current
|
||||
directory, active search) and ``recent_files`` the last modified files, so
|
||||
the assistant knows what the user is doing rather than answering blind.
|
||||
|
||||
``agent`` selects the action protocol: the classic chat endpoint (no native
|
||||
tools) uses the text ``obsigate-action`` blocks, while the tool-calling
|
||||
agent endpoint must invoke the native tools instead (BUG-053).
|
||||
"""
|
||||
protocol = GENERAL_ACTION_TOOL_PROTOCOL if agent else GENERAL_ACTION_TEXT_PROTOCOL
|
||||
prompt = GENERAL_SYSTEM_HEADER + protocol
|
||||
if vaults:
|
||||
prompt += "\nVaults disponibles : " + ", ".join(sorted(vaults)) + "\n"
|
||||
else:
|
||||
prompt += "\nAucun vault n'est actuellement configuré.\n"
|
||||
block = _format_app_context(app_context, recent_files)
|
||||
if block:
|
||||
prompt += "\n" + block
|
||||
return prompt
|
||||
|
||||
|
||||
def invalidate_cache(vault_path: Path | None = None, directory: str | None = None) -> int:
|
||||
"""Invalidate cache entries.
|
||||
|
||||
Args:
|
||||
vault_path: If provided, only invalidate entries for this vault.
|
||||
directory: If provided, only invalidate entries for this directory.
|
||||
|
||||
Returns:
|
||||
Number of cache entries removed.
|
||||
"""
|
||||
if vault_path is None and directory is None:
|
||||
count = len(_cache)
|
||||
_cache.clear()
|
||||
return count
|
||||
|
||||
to_remove = list(_cache)
|
||||
for k in to_remove:
|
||||
del _cache[k]
|
||||
return len(to_remove)
|
||||
@@ -0,0 +1,722 @@
|
||||
"""BooksLM API routes — directory-scoped AI chat for Obsidian vaults."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from backend.agent.loop import run_agent
|
||||
from backend.ai_chat import chat_completion, stream_completion
|
||||
from backend.ai_history import delete_session, get_session, list_sessions, upsert_session
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.bookslm import (
|
||||
build_general_system_prompt,
|
||||
build_system_prompt,
|
||||
collect_adhoc_context,
|
||||
collect_directory_context,
|
||||
collect_files_context,
|
||||
empty_context,
|
||||
load_vault_image_data_url,
|
||||
merge_contexts,
|
||||
)
|
||||
from backend.indexer import get_vault_data, index
|
||||
from backend.model_capabilities import model_supports_vision
|
||||
from backend.schemas import BooksLMContextResponse
|
||||
from backend.skills import get_skill_prompt
|
||||
from backend.tools.api import ToolContext, ToolMode
|
||||
|
||||
logger = logging.getLogger("obsigate.bookslm_routes")
|
||||
router = APIRouter(prefix="/api/ai/bookslm", tags=["BooksLM"])
|
||||
|
||||
VALID_MODES = {"directory", "documents", "general"}
|
||||
|
||||
|
||||
# ── Request models ──
|
||||
|
||||
|
||||
class BooksLMContextRequest(BaseModel):
|
||||
vault: str | None = Field(default=None, description="Vault name (required for directory/documents modes)")
|
||||
directory: str = Field(default="", description="Relative directory path within the vault")
|
||||
mode: str = Field(default="directory", description="Context mode: 'directory', 'documents' or 'general'")
|
||||
context_files: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Relative file paths to use as context (documents mode)",
|
||||
)
|
||||
extra_files: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Ad-hoc files added with the '@' command (any mode)",
|
||||
)
|
||||
extra_directories: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Ad-hoc directories added with the '@' command (any mode)",
|
||||
)
|
||||
app_context: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Live client UI state for the General assistant: open_documents, "
|
||||
"current_path, directory, vault, search_query, search_total, search_results.",
|
||||
)
|
||||
|
||||
|
||||
class BooksLMChatRequest(BaseModel):
|
||||
vault: str | None = Field(default=None, description="Vault name (required for directory/documents modes)")
|
||||
directory: str = Field(default="", description="Relative directory path within the vault")
|
||||
mode: str = Field(default="directory", description="Context mode: 'directory', 'documents' or 'general'")
|
||||
context_files: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Relative file paths to use as context (documents mode)",
|
||||
)
|
||||
extra_files: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Ad-hoc files added with the '@' command (any mode)",
|
||||
)
|
||||
extra_directories: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Ad-hoc directories added with the '@' command (any mode)",
|
||||
)
|
||||
message: str = Field(description="User message")
|
||||
images: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Images for vision models. Each item: {data, mime_type} (pasted) "
|
||||
"or {path} (vault-relative file).",
|
||||
)
|
||||
skill: str | None = Field(
|
||||
default=None,
|
||||
description="Skill id selected with the '/' command; its prompt is added to the system prompt.",
|
||||
)
|
||||
conversation_history: list[dict[str, str]] = Field(
|
||||
default_factory=list,
|
||||
description="Previous conversation turns [{role, content}]",
|
||||
)
|
||||
provider: str | None = Field(
|
||||
default=None,
|
||||
description="AI provider override (e.g. 'deepseek', 'openrouter', 'gemini', 'nvidia', 'xiaomi', 'mistral', 'qwencloud'). "
|
||||
"If not set, uses DEFAULT_PROVIDER.",
|
||||
)
|
||||
model: str | None = Field(
|
||||
default=None,
|
||||
description="Model name to use for this request. If not set, uses the provider's default model.",
|
||||
)
|
||||
confirm: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Pending tool confirmation to apply (two-step propose/apply). "
|
||||
"Shape: the ``error`` object of a previous ``confirmation`` event.",
|
||||
)
|
||||
confirm_messages: list[dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description="Conversation snapshot returned alongside a ``confirmation`` event, "
|
||||
"echoed back to resume the agent run.",
|
||||
)
|
||||
confirm_all: bool = Field(
|
||||
default=False,
|
||||
description="Global approval (BUG-075): apply every pending action of the batch "
|
||||
"and auto-approve the remaining mutating calls of the same run, "
|
||||
"so the run does not pause on each action.",
|
||||
)
|
||||
app_context: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Live client UI state for the General assistant: open_documents, "
|
||||
"current_path, directory, vault, search_query, search_total, search_results.",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_mode(mode: str | None) -> str:
|
||||
mode = (mode or "directory").lower()
|
||||
return mode if mode in VALID_MODES else "directory"
|
||||
|
||||
|
||||
def _resolve_vault_path(vault: str | None, current_user):
|
||||
"""Validate vault access and return (vault_name, vault_path)."""
|
||||
if not vault:
|
||||
raise HTTPException(status_code=400, detail="Champ 'vault' requis pour ce contexte")
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
vault_data = get_vault_data(vault)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault}' not found")
|
||||
return vault, Path(vault_data["path"])
|
||||
|
||||
|
||||
def _build_context(
|
||||
mode: str,
|
||||
vault_path: Path | None,
|
||||
directory: str,
|
||||
context_files: list[str],
|
||||
extra_files: list[str] | None = None,
|
||||
extra_directories: list[str] | None = None,
|
||||
):
|
||||
"""Collect the context payload for the requested mode.
|
||||
|
||||
Ad-hoc files/directories (``@`` command) are merged on top of the base
|
||||
context. In General mode, attaching ad-hoc files promotes the effective
|
||||
scope to ``documents`` so the prompt actually includes their content.
|
||||
"""
|
||||
if mode == "general":
|
||||
base = empty_context("general")
|
||||
elif mode == "documents":
|
||||
ctx = collect_files_context(vault_path, context_files, scope="documents") # type: ignore[arg-type]
|
||||
# No open document could be read → degrade gracefully to General.
|
||||
base = ctx if ctx["file_count"] else empty_context("general")
|
||||
else:
|
||||
base = collect_directory_context(vault_path, directory) # type: ignore[arg-type]
|
||||
|
||||
if (extra_files or extra_directories) and vault_path is not None:
|
||||
adhoc_scope = "documents" if base.get("scope") == "general" else base.get("scope", "directory")
|
||||
adhoc = collect_adhoc_context(vault_path, extra_files, extra_directories, scope=adhoc_scope)
|
||||
if adhoc["file_count"]:
|
||||
base = merge_contexts(base, adhoc)
|
||||
return base
|
||||
|
||||
|
||||
def _submitted_app_context(req) -> dict[str, Any] | None:
|
||||
"""Return the client-submitted live app state (may be None)."""
|
||||
ctx = getattr(req, "app_context", None)
|
||||
return ctx if isinstance(ctx, dict) else None
|
||||
|
||||
|
||||
def _recent_files_for_prompt(current_user, limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Best-effort list of the user's most recently modified files.
|
||||
|
||||
Used to give the General assistant a sense of what the user has been
|
||||
working on. Never raises: any failure yields an empty list.
|
||||
"""
|
||||
try:
|
||||
from backend.services.recent import list_recent
|
||||
|
||||
username = current_user.get("username") if isinstance(current_user, dict) else None
|
||||
user_vaults = (
|
||||
current_user.get("_token_vaults") or current_user.get("vaults", [])
|
||||
if isinstance(current_user, dict)
|
||||
else []
|
||||
)
|
||||
data = list_recent(username, user_vaults, limit=limit, mode="modified")
|
||||
return list(data.get("files", []))
|
||||
except Exception: # pragma: no cover - defensive, prompt enrichment only
|
||||
logger.debug("Could not gather recent files for assistant prompt", exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
def _resolve_system_prompt(req, current_user, agent: bool = False) -> str:
|
||||
"""Resolve the vault access and build the assistant system prompt.
|
||||
|
||||
Shared by the classic chat endpoint and the tool-calling agent endpoint.
|
||||
``agent=True`` selects the native-tool action protocol (no text
|
||||
``obsigate-action`` blocks) for the General/empty-directory prompts.
|
||||
"""
|
||||
mode = _normalize_mode(req.mode)
|
||||
vault_path: Path | None = None
|
||||
if mode != "general":
|
||||
_, vault_path = _resolve_vault_path(req.vault, current_user)
|
||||
elif getattr(req, "extra_files", None) or getattr(req, "extra_directories", None):
|
||||
# General mode has no base context, but ad-hoc files/directories added
|
||||
# with `@` still need a vault to be read from.
|
||||
vault_path = _resolve_optional_vault_path(req, current_user)
|
||||
|
||||
context = _build_context(
|
||||
mode,
|
||||
vault_path,
|
||||
req.directory,
|
||||
req.context_files,
|
||||
getattr(req, "extra_files", None),
|
||||
getattr(req, "extra_directories", None),
|
||||
)
|
||||
effective_mode = context.get("scope", mode)
|
||||
|
||||
if effective_mode == "general":
|
||||
prompt = build_general_system_prompt(
|
||||
list(index.keys()),
|
||||
app_context=_submitted_app_context(req),
|
||||
recent_files=_recent_files_for_prompt(current_user),
|
||||
agent=agent,
|
||||
)
|
||||
elif effective_mode == "documents":
|
||||
prompt = build_system_prompt(context, scope="documents", vault_name=req.vault)
|
||||
elif context["file_count"] == 0:
|
||||
# Empty (or unreadable) directory: don't block the request. Answer as
|
||||
# the General assistant would, telling the model the folder is empty so
|
||||
# it can still help (create a file, explain the app, etc.).
|
||||
prompt = build_general_system_prompt(
|
||||
list(index.keys()),
|
||||
app_context=_submitted_app_context(req),
|
||||
recent_files=_recent_files_for_prompt(current_user),
|
||||
agent=agent,
|
||||
)
|
||||
prompt += (
|
||||
f"\n## Dossier vide\nLe dossier « {req.directory or '/'} » "
|
||||
f"(vault {req.vault}) ne contient aucun fichier markdown exploitable. "
|
||||
"Réponds quand même à la demande de l'utilisateur sans contexte "
|
||||
"documentaire, et propose une action de création si c'est pertinent.\n"
|
||||
)
|
||||
else:
|
||||
prompt = build_system_prompt(context, scope="directory", vault_name=req.vault)
|
||||
|
||||
if agent and effective_mode != "general" and context["file_count"] > 0:
|
||||
prompt += (
|
||||
"\n## Mode agent\n"
|
||||
"Utilise les outils natifs (function calling) pour agir sur les fichiers "
|
||||
"(`create_file`, `create_directory`, `append_to_file`, `edit_file`, …). "
|
||||
"N'écris jamais de bloc ```obsigate-action```."
|
||||
)
|
||||
|
||||
skill_id = getattr(req, "skill", None)
|
||||
if skill_id:
|
||||
skill_prompt = get_skill_prompt(skill_id, current_user)
|
||||
if skill_prompt:
|
||||
prompt += "\n\n## Skill actif\n" + skill_prompt
|
||||
return prompt
|
||||
|
||||
|
||||
def _resolve_optional_vault_path(req, current_user) -> Path | None:
|
||||
"""Best-effort vault path resolution (never raises)."""
|
||||
if not getattr(req, "vault", None):
|
||||
return None
|
||||
try:
|
||||
_, vault_path = _resolve_vault_path(req.vault, current_user)
|
||||
return vault_path
|
||||
except HTTPException:
|
||||
return None
|
||||
|
||||
|
||||
def _build_user_content(req, vault_path: Path | None):
|
||||
"""Build the user message content, attaching images when present.
|
||||
|
||||
Returns a plain string when there is no image, otherwise an OpenAI-style
|
||||
multimodal content array (which ``ai_chat`` adapts for Gemini).
|
||||
"""
|
||||
images = getattr(req, "images", None) or []
|
||||
if not images:
|
||||
return req.message
|
||||
|
||||
parts: list[dict[str, Any]] = [{"type": "text", "text": req.message}]
|
||||
for image in images:
|
||||
if not isinstance(image, dict):
|
||||
continue
|
||||
data_url: str | None = None
|
||||
if image.get("data"):
|
||||
mime = image.get("mime_type") or "image/png"
|
||||
data_url = f"data:{mime};base64,{image['data']}"
|
||||
elif image.get("path") and vault_path is not None:
|
||||
data_url = load_vault_image_data_url(vault_path, str(image["path"]))
|
||||
if data_url:
|
||||
parts.append({"type": "image_url", "image_url": {"url": data_url}})
|
||||
return parts
|
||||
|
||||
|
||||
def _validate_vision_support(req) -> None:
|
||||
"""Reject image requests when the selected model cannot analyse images."""
|
||||
if not (getattr(req, "images", None)):
|
||||
return
|
||||
provider = _resolve_provider_name(req.provider)
|
||||
if not provider:
|
||||
return
|
||||
from backend.ai import PROVIDERS
|
||||
|
||||
model = req.model or PROVIDERS.get(provider, {}).get("model", "")
|
||||
if not model_supports_vision(provider, model):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Le modèle '{model or provider}' ne supporte pas l'analyse d'images. "
|
||||
"Choisissez un modèle compatible vision.",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_provider_name(requested: str | None) -> str | None:
|
||||
"""Pick the provider to use: explicit override, else first available."""
|
||||
from backend.ai import DEFAULT_PROVIDER, PROVIDERS
|
||||
|
||||
cfg_name = (requested or DEFAULT_PROVIDER).lower()
|
||||
if cfg_name in PROVIDERS and PROVIDERS[cfg_name].get("api_key"):
|
||||
return cfg_name
|
||||
for pname, pcfg in PROVIDERS.items():
|
||||
if pcfg.get("api_key") and pname != "gemini":
|
||||
return pname
|
||||
return None
|
||||
|
||||
|
||||
def _effective_model(provider: str | None, requested: str | None) -> str:
|
||||
"""Model actually used for a request.
|
||||
|
||||
The client may leave `model` empty (provider default) — reporting the raw
|
||||
request would show nothing in the "provider · model" tag, so the provider's
|
||||
configured default is returned instead.
|
||||
"""
|
||||
if requested:
|
||||
return requested
|
||||
if not provider:
|
||||
return ""
|
||||
from backend.ai import PROVIDERS
|
||||
|
||||
return PROVIDERS.get(provider, {}).get("model", "") or ""
|
||||
|
||||
|
||||
def _tool_sources(rec) -> list[dict[str, str]]:
|
||||
"""Compact web sources of a tool result (rendered as links in the UI).
|
||||
|
||||
Only the web tools produce sources: ``web_search`` returns ranked results,
|
||||
``fetch_url`` a single page. Everything else yields an empty list so the
|
||||
SSE payload stays small.
|
||||
"""
|
||||
data = rec.result if isinstance(rec.result, dict) else {}
|
||||
name = rec.name or ""
|
||||
sources: list[dict[str, str]] = []
|
||||
if name == "web_search":
|
||||
for item in (data.get("results") or [])[:8]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = item.get("url") or ""
|
||||
if not url:
|
||||
continue
|
||||
sources.append({"title": item.get("title") or url, "url": url})
|
||||
elif name == "fetch_url":
|
||||
url = data.get("url") or ""
|
||||
if url:
|
||||
sources.append({"title": data.get("title") or url, "url": url})
|
||||
return sources
|
||||
|
||||
|
||||
def _tool_event_sse(rec) -> str:
|
||||
"""Serialize one executed tool call as an SSE ``tool`` event."""
|
||||
payload = json.dumps(
|
||||
{
|
||||
"name": rec.name,
|
||||
"ok": rec.ok,
|
||||
"arguments": rec.arguments,
|
||||
"step": rec.step,
|
||||
"sources": _tool_sources(rec),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return f"event: tool\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
def _thought_event_sse(note: dict) -> str:
|
||||
"""Serialize one intermediate reasoning note as an SSE ``step`` event."""
|
||||
payload = json.dumps({"step": note}, ensure_ascii=False)
|
||||
return f"event: step\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
# ── Endpoints ──
|
||||
|
||||
|
||||
@router.post("/context", response_model=BooksLMContextResponse)
|
||||
async def api_bookslm_context(
|
||||
req: BooksLMContextRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Collect context for the AI assistant.
|
||||
|
||||
Three modes are supported:
|
||||
|
||||
* ``directory`` — every supported file under a vault directory;
|
||||
* ``documents`` — only the explicitly listed open documents;
|
||||
* ``general`` — no document context (assistant for the app itself).
|
||||
"""
|
||||
mode = _normalize_mode(req.mode)
|
||||
if mode == "general":
|
||||
if not (req.extra_files or req.extra_directories):
|
||||
return empty_context("general")
|
||||
vault_path = _resolve_optional_vault_path(req, current_user)
|
||||
else:
|
||||
_, vault_path = _resolve_vault_path(req.vault, current_user)
|
||||
return _build_context(
|
||||
mode, vault_path, req.directory, req.context_files,
|
||||
req.extra_files, req.extra_directories,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/chat",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"text/event-stream": {}}, "description": "SSE token stream"}},
|
||||
)
|
||||
async def api_bookslm_chat(
|
||||
req: BooksLMChatRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Chat with the AI assistant.
|
||||
|
||||
Builds a system prompt from the selected context (directory, open
|
||||
documents or general app knowledge), then streams the provider's answer
|
||||
as Server-Sent Events.
|
||||
"""
|
||||
_validate_vision_support(req)
|
||||
system_prompt = _resolve_system_prompt(req, current_user)
|
||||
vault_path = _resolve_optional_vault_path(req, current_user)
|
||||
|
||||
messages: list[dict[str, Any]] = [{"role": "system", "content": system_prompt}]
|
||||
for turn in req.conversation_history:
|
||||
role = turn.get("role")
|
||||
content = turn.get("content", "")
|
||||
if role in ("user", "assistant") and content:
|
||||
messages.append({"role": role, "content": content})
|
||||
messages.append({"role": "user", "content": _build_user_content(req, vault_path)})
|
||||
|
||||
async def generate_sse():
|
||||
try:
|
||||
# Resolve provider: explicit override wins, else first available.
|
||||
cfg_name = _resolve_provider_name(req.provider)
|
||||
if cfg_name is None:
|
||||
err = "Aucun fournisseur AI configuré (clés API manquantes)"
|
||||
error_data = json.dumps({"error": err}, ensure_ascii=False)
|
||||
yield f"event: error\ndata: {error_data}\n\n"
|
||||
return
|
||||
|
||||
# Stream token deltas as they arrive from the provider.
|
||||
async for token in stream_completion(
|
||||
messages,
|
||||
provider=cfg_name,
|
||||
model=req.model,
|
||||
temperature=0.3,
|
||||
max_tokens=4096,
|
||||
):
|
||||
data = json.dumps(
|
||||
{
|
||||
"token": token,
|
||||
"provider": cfg_name,
|
||||
"model": _effective_model(cfg_name, req.model),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
yield f"event: message\ndata: {data}\n\n"
|
||||
yield "event: done\ndata: {}\n\n"
|
||||
except Exception as e:
|
||||
logger.error(f"BooksLM chat error: {e}")
|
||||
error_data = json.dumps({"error": str(e)}, ensure_ascii=False)
|
||||
yield f"event: error\ndata: {error_data}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
generate_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/agent",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"text/event-stream": {}}, "description": "SSE tool/agent stream"}},
|
||||
)
|
||||
async def api_bookslm_agent(
|
||||
req: BooksLMChatRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Chat with the tool-calling agent.
|
||||
|
||||
Same context as ``/chat`` but the model may call tools (read/search the
|
||||
vault) through the shared tool layer. Emits one ``tool`` event per executed
|
||||
tool call, then a final ``message`` event. Mutating tools pause the run with
|
||||
a ``confirmation`` event (two-step propose/apply) carrying the pending
|
||||
``actions`` (every mutating call of the turn) and the conversation snapshot;
|
||||
the client resumes by echoing them back in ``confirm`` / ``confirm_messages``,
|
||||
optionally with ``confirm_all`` to apply the whole batch and auto-approve the
|
||||
rest of the run (BUG-075).
|
||||
"""
|
||||
_validate_vision_support(req)
|
||||
system_prompt = _resolve_system_prompt(req, current_user, agent=True)
|
||||
vault_path = _resolve_optional_vault_path(req, current_user)
|
||||
|
||||
messages: list[dict] = [{"role": "system", "content": system_prompt}]
|
||||
for turn in req.conversation_history:
|
||||
role = turn.get("role")
|
||||
content = turn.get("content", "")
|
||||
if role in ("user", "assistant") and content:
|
||||
messages.append({"role": role, "content": content})
|
||||
messages.append({"role": "user", "content": _build_user_content(req, vault_path)})
|
||||
|
||||
ctx = ToolContext(user=current_user, mode=ToolMode.IN_APP)
|
||||
if req.confirm_all:
|
||||
# BUG-075: a single global approval authorizes the whole plan, so the
|
||||
# run no longer pauses on every subsequent mutating call.
|
||||
ctx.confirmed = True
|
||||
|
||||
async def generate_sse():
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
cfg_name = _resolve_provider_name(req.provider)
|
||||
if cfg_name is None:
|
||||
error_data = json.dumps(
|
||||
{"error": "Aucun fournisseur AI configuré (clés API manquantes)"},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
yield f"event: error\ndata: {error_data}\n\n"
|
||||
return
|
||||
|
||||
# #187: resolve the provider like /chat does — the agent must use
|
||||
# the same engine the SSE "provider" tag reports (the raw
|
||||
# req.provider could name an unavailable provider and silently
|
||||
# fall back to another one via _get_provider_config).
|
||||
async def _llm(msgs, tool_schemas):
|
||||
return await chat_completion(
|
||||
msgs,
|
||||
tools=tool_schemas,
|
||||
provider=cfg_name,
|
||||
model=req.model,
|
||||
temperature=0.3,
|
||||
# Tool-call arguments can carry a whole file body (e.g. a
|
||||
# generated table): leave more room than the plain-chat
|
||||
# default.
|
||||
max_tokens=8192,
|
||||
)
|
||||
|
||||
# Stream tool events live: each executed step is pushed on the
|
||||
# queue by the loop callback and emitted as soon as it happens,
|
||||
# so the UI can grow its « N steps » block while thinking.
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
def _on_tool(rec) -> None:
|
||||
queue.put_nowait(("tool", rec))
|
||||
|
||||
run_task = asyncio.create_task(run_agent(
|
||||
messages,
|
||||
ctx=ctx,
|
||||
llm=_llm,
|
||||
resume_messages=req.confirm_messages,
|
||||
confirm_pending=req.confirm,
|
||||
on_tool_call=_on_tool,
|
||||
on_thought=lambda note: queue.put_nowait(("thought", note)),
|
||||
))
|
||||
|
||||
# Drain every completed step as soon as it lands, while the agent
|
||||
# keeps running in the background. If the client disconnects, the
|
||||
# generator is cancelled: release the run so it cannot orphan.
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
kind, item = await asyncio.wait_for(queue.get(), timeout=0.25)
|
||||
except TimeoutError:
|
||||
if run_task.done():
|
||||
break
|
||||
continue
|
||||
yield _tool_event_sse(item) if kind == "tool" else _thought_event_sse(item)
|
||||
while not queue.empty():
|
||||
kind, item = queue.get_nowait()
|
||||
yield _tool_event_sse(item) if kind == "tool" else _thought_event_sse(item)
|
||||
result = run_task.result()
|
||||
finally:
|
||||
if not run_task.done():
|
||||
run_task.cancel()
|
||||
|
||||
if result.stopped == "confirmation_required":
|
||||
pending = json.dumps(
|
||||
{"pending": result.pending or {}, "messages": result.messages},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
yield f"event: confirmation\ndata: {pending}\n\n"
|
||||
else:
|
||||
data = json.dumps(
|
||||
{
|
||||
"token": result.content,
|
||||
"provider": cfg_name,
|
||||
"model": _effective_model(cfg_name, req.model),
|
||||
"iterations": result.iterations,
|
||||
"stopped": result.stopped,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
yield f"event: message\ndata: {data}\n\n"
|
||||
yield "event: done\ndata: {}\n\n"
|
||||
except Exception as e:
|
||||
logger.error(f"BooksLM agent error: {e}")
|
||||
error_data = json.dumps({"error": str(e)}, ensure_ascii=False)
|
||||
yield f"event: error\ndata: {error_data}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
generate_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── Persistent conversation history (#95) ─────────────────────────────
|
||||
|
||||
|
||||
class BookslmSession(BaseModel):
|
||||
"""Full assistant conversation persisted server-side (#95).
|
||||
|
||||
The shape mirrors the client session object so round-tripping is lossless
|
||||
(transient fields such as ``confirmation``/``payload`` are stripped
|
||||
client-side before upload).
|
||||
"""
|
||||
|
||||
id: str = Field(description="Stable session id (s-<ts36>-<rand>)")
|
||||
title: str = Field(default="", description="Derived human title")
|
||||
mode: str = Field(default="general", description="'directory', 'documents' or 'general'")
|
||||
vault: str | None = Field(default=None, description="Vault name for the context")
|
||||
directory: str = Field(default="", description="Relative directory path")
|
||||
documents: list[dict[str, str]] = Field(
|
||||
default_factory=list,
|
||||
description="Open documents for the 'documents' mode",
|
||||
)
|
||||
context: str = Field(default="", description="Client context key of the assistant")
|
||||
createdAt: int | None = Field(default=None, description="Creation ISO ms timestamp")
|
||||
updatedAt: int | None = Field(default=None, description="Last update ISO ms timestamp")
|
||||
messages: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Conversation turns [{role, content}]",
|
||||
)
|
||||
|
||||
|
||||
def _session_user(current_user) -> str:
|
||||
return current_user.get("username") if isinstance(current_user, dict) else "" # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.get("/history", response_model=dict[str, list[dict[str, Any]]])
|
||||
async def api_bookslm_history_list(current_user=Depends(require_auth)):
|
||||
"""List the user's assistant conversations (summaries, most recent first).
|
||||
|
||||
Messages are not included to keep the list light; fetch the full
|
||||
conversation with ``GET /history/{id}`` when a session is opened.
|
||||
"""
|
||||
return {"sessions": list_sessions(_session_user(current_user))}
|
||||
|
||||
|
||||
@router.get("/history/{session_id}", response_model=dict[str, Any] | None)
|
||||
async def api_bookslm_history_get(
|
||||
session_id: str,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return a single full conversation, or 404 when unknown."""
|
||||
session = get_session(_session_user(current_user), session_id)
|
||||
if session is None:
|
||||
raise HTTPException(status_code=404, detail="Conversation not found")
|
||||
return session
|
||||
|
||||
|
||||
@router.put("/history/{session_id}", response_model=dict[str, Any])
|
||||
async def api_bookslm_history_upsert(
|
||||
session_id: str,
|
||||
session: BookslmSession,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Create or update a conversation for the current user.
|
||||
|
||||
The body id is forced to the path id so a client never writes under a
|
||||
different key by mistake.
|
||||
"""
|
||||
payload = session.model_dump()
|
||||
payload["id"] = session_id
|
||||
stored = upsert_session(_session_user(current_user), payload)
|
||||
if stored is None:
|
||||
raise HTTPException(status_code=400, detail="Invalid session id")
|
||||
return stored
|
||||
|
||||
|
||||
@router.delete("/history/{session_id}", response_model=dict[str, bool])
|
||||
async def api_bookslm_history_delete(
|
||||
session_id: str,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Delete a conversation for the current user."""
|
||||
return {"ok": delete_session(_session_user(current_user), session_id)}
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Chiffrement par enveloppe du coffre Secrets ObsiGate (Phase 1a).
|
||||
|
||||
Principe (spec §5) : deux niveaux de clés, jamais de valeur en clair au repos.
|
||||
|
||||
- KEK : clé maître 256 bits, fournie par l'opérateur via ``OBSIGATE_KEK``
|
||||
(base64, 32 octets) ou ``OBSIGATE_KEK_FILE`` (fichier 0600, hors dépôt,
|
||||
hors arborescence des vaults). Elle n'est JAMAIS dérivée du secret
|
||||
d'application, jamais journalisée, jamais renvoyée par API.
|
||||
- DEK : une clé aléatoire par coffre, stockée uniquement chiffrée par la KEK.
|
||||
- Version de secret : ciphertext AES-256-GCM + nonce unique + tag GCM
|
||||
(le tag est inclus dans le ciphertext renvoyé par ``AESGCM.encrypt``).
|
||||
Un tag invalide ou un fichier altéré = refus, jamais de valeur fausse.
|
||||
|
||||
Le module reste importable sans ``cryptography`` installé : les opérations
|
||||
échouent alors avec ``CryptoIndisponibleError`` au lieu de casser le
|
||||
démarrage de l'application (le routeur répond 503 dans ce cas).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.coffre_crypto")
|
||||
|
||||
# Annotation explicite : sans elle, mypy refuse l'assignation de deux types
|
||||
# (la classe AESGCM importée, puis None dans le repli sans cryptography)
|
||||
# et le commentaire `type: ignore[assignment]` ne couvre pas ce code [misc].
|
||||
AESGCM: type[Any] | None
|
||||
try: # cryptography est une dépendance de la Phase 1a (requirements.txt)
|
||||
from cryptography.exceptions import InvalidTag
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
except Exception: # pragma: no cover - environnement sans cryptography
|
||||
AESGCM = None
|
||||
InvalidTag = Exception # type: ignore[assignment,misc]
|
||||
|
||||
|
||||
class CoffreCryptoError(Exception):
|
||||
"""Erreur générique du chiffrement du coffre."""
|
||||
|
||||
|
||||
class KEKAbsenteError(CoffreCryptoError):
|
||||
"""Aucune KEK configurée (ni OBSIGATE_KEK, ni OBSIGATE_KEK_FILE)."""
|
||||
|
||||
|
||||
class KEKInvalideError(CoffreCryptoError):
|
||||
"""KEK mal formée (ni base64 32 octets, ni hex 64 caractères)."""
|
||||
|
||||
|
||||
class TagInvalideError(CoffreCryptoError):
|
||||
"""Authentification GCM échouée : données altérées ou mauvaise clé."""
|
||||
|
||||
|
||||
class CryptoIndisponibleError(CoffreCryptoError):
|
||||
"""La bibliothèque ``cryptography`` n'est pas installée."""
|
||||
|
||||
|
||||
def _require_aesgcm() -> type[Any]:
|
||||
"""Retourne la classe AESGCM, ou lève si cryptography est absent.
|
||||
|
||||
Retourner la classe (plutôt qu'un simple garde) permet à mypy de
|
||||
savoir que l'appelant détient bien un constructible, et non None.
|
||||
"""
|
||||
if AESGCM is None:
|
||||
raise CryptoIndisponibleError(
|
||||
"cryptography n'est pas installé (dépendance Phase 1a)"
|
||||
)
|
||||
return AESGCM
|
||||
|
||||
|
||||
def parse_kek(raw: str) -> bytes:
|
||||
"""Parse une KEK : base64 (32 octets) ou hex (64 caractères).
|
||||
|
||||
Accepte le base64 standard et URL-safe, avec ou sans padding.
|
||||
"""
|
||||
value = (raw or "").strip()
|
||||
if not value:
|
||||
raise KEKInvalideError("KEK vide")
|
||||
# Hex d'abord si la forme correspond (64 caractères hex).
|
||||
if len(value) == 64:
|
||||
try:
|
||||
decoded = bytes.fromhex(value)
|
||||
except ValueError:
|
||||
decoded = b""
|
||||
if len(decoded) == 32:
|
||||
return decoded
|
||||
padded = value + "=" * (-len(value) % 4)
|
||||
for decoder in (base64.urlsafe_b64decode, base64.b64decode):
|
||||
try:
|
||||
decoded = decoder(padded)
|
||||
except (binascii.Error, ValueError):
|
||||
continue
|
||||
if len(decoded) == 32:
|
||||
return decoded
|
||||
raise KEKInvalideError("KEK invalide : attendu base64 de 32 octets")
|
||||
|
||||
|
||||
def generate_kek_b64() -> str:
|
||||
"""Génère une KEK 256 bits, encodée base64 (à sauvegarder hors dépôt)."""
|
||||
return base64.b64encode(os.urandom(32)).decode("ascii")
|
||||
|
||||
|
||||
def load_kek() -> bytes:
|
||||
"""Charge la KEK depuis l'environnement ou le fichier dédié.
|
||||
|
||||
Priorité : ``OBSIGATE_KEK`` puis ``OBSIGATE_KEK_FILE``. Le fichier doit
|
||||
être en 0600 (aucun droit groupe/autre) ; sinon, refus explicite.
|
||||
"""
|
||||
env_kek = os.environ.get("OBSIGATE_KEK", "").strip()
|
||||
if env_kek:
|
||||
return parse_kek(env_kek)
|
||||
kek_file = os.environ.get("OBSIGATE_KEK_FILE", "").strip()
|
||||
if kek_file:
|
||||
path = Path(kek_file)
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
except OSError as exc:
|
||||
raise KEKAbsenteError(f"Fichier KEK illisible : {kek_file}") from exc
|
||||
if mode & (stat.S_IRWXG | stat.S_IRWXO):
|
||||
raise KEKInvalideError(
|
||||
"Fichier KEK trop permissif : attendu 0600, "
|
||||
f"droits groupe/autre détectés sur {kek_file}"
|
||||
)
|
||||
try:
|
||||
return parse_kek(path.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise KEKAbsenteError(f"Fichier KEK illisible : {kek_file}") from exc
|
||||
raise KEKAbsenteError("Aucune KEK configurée (OBSIGATE_KEK / OBSIGATE_KEK_FILE)")
|
||||
|
||||
|
||||
def kek_presente() -> bool:
|
||||
"""True si une KEK valide est chargeable, False sinon (jamais d'exception)."""
|
||||
try:
|
||||
load_kek()
|
||||
except CoffreCryptoError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def generate_dek() -> bytes:
|
||||
"""Génère une DEK aléatoire 256 bits pour un coffre."""
|
||||
return os.urandom(32)
|
||||
|
||||
|
||||
def _b64(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii")
|
||||
|
||||
|
||||
def _unb64(value: str) -> bytes:
|
||||
return base64.b64decode(value)
|
||||
|
||||
|
||||
def encrypt_with_key(key: bytes, plaintext: bytes, aad: bytes) -> dict:
|
||||
"""Chiffre avec AES-256-GCM. Retourne {nonce, ciphertext} en base64."""
|
||||
aesgcm = _require_aesgcm()
|
||||
nonce = os.urandom(12)
|
||||
ciphertext = aesgcm(key).encrypt(nonce, plaintext, aad)
|
||||
return {"nonce": _b64(nonce), "ciphertext": _b64(ciphertext)}
|
||||
|
||||
|
||||
def decrypt_with_key(key: bytes, nonce_b64: str, ciphertext_b64: str, aad: bytes) -> bytes:
|
||||
"""Déchiffre avec AES-256-GCM. Tag invalide -> TagInvalideError."""
|
||||
aesgcm = _require_aesgcm()
|
||||
try:
|
||||
return aesgcm(key).decrypt(_unb64(nonce_b64), _unb64(ciphertext_b64), aad)
|
||||
except InvalidTag as exc:
|
||||
raise TagInvalideError("Tag GCM invalide : données altérées ou mauvaise clé") from exc
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise TagInvalideError("Données chiffrées mal formées") from exc
|
||||
|
||||
|
||||
def wrap_dek(dek: bytes, kek: bytes, coffre_id: str) -> dict:
|
||||
"""Chiffre une DEK avec la KEK (AAD = identifiant du coffre)."""
|
||||
return encrypt_with_key(kek, dek, coffre_id.encode("utf-8"))
|
||||
|
||||
|
||||
def unwrap_dek(wrapped: dict, kek: bytes, coffre_id: str) -> bytes:
|
||||
"""Déchiffre une DEK avec la KEK."""
|
||||
return decrypt_with_key(
|
||||
kek, wrapped["nonce"], wrapped["ciphertext"], coffre_id.encode("utf-8")
|
||||
)
|
||||
|
||||
|
||||
def encrypt_secret_value(dek: bytes, secret_id: str, version: int, valeur: str) -> dict:
|
||||
"""Chiffre une valeur de secret (AAD = secret_id:version)."""
|
||||
aad = f"{secret_id}:{version}".encode()
|
||||
return encrypt_with_key(dek, valeur.encode("utf-8"), aad)
|
||||
|
||||
|
||||
def decrypt_secret_value(
|
||||
dek: bytes, secret_id: str, version: int, nonce_b64: str, ciphertext_b64: str
|
||||
) -> str:
|
||||
"""Déchiffre une valeur de secret."""
|
||||
aad = f"{secret_id}:{version}".encode()
|
||||
return decrypt_with_key(dek, nonce_b64, ciphertext_b64, aad).decode("utf-8")
|
||||
|
||||
|
||||
def reencrypt_secret_value(
|
||||
ancienne_dek: bytes,
|
||||
nouvelle_dek: bytes,
|
||||
secret_id: str,
|
||||
version: int,
|
||||
nonce_b64: str,
|
||||
ciphertext_b64: str,
|
||||
) -> dict:
|
||||
"""Re-chiffre une version avec une nouvelle DEK (rotation DEK)."""
|
||||
valeur = decrypt_secret_value(
|
||||
ancienne_dek, secret_id, version, nonce_b64, ciphertext_b64
|
||||
)
|
||||
return encrypt_secret_value(nouvelle_dek, secret_id, version, valeur)
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Migration Phase 2 du coffre Secrets ObsiGate (issue #20).
|
||||
|
||||
Bascule des clés aujourd'hui en clair dans ``data/api_keys.json`` vers le
|
||||
coffre d'instance « ObsiGate interne » (slug ``obsigate-interne``).
|
||||
|
||||
Format source (constaté dans ``backend/ai.py`` et
|
||||
``backend/tools/secrets.py``) : un dictionnaire plat ``{NOM: valeur}``.
|
||||
Seules les entrées dont le nom ressemble à un secret sont migrables :
|
||||
clés IA et clés d'outils. Les valeurs non secrètes (par exemple
|
||||
``OBSIGATE_GITEA_URL``) ne sont JAMAIS migrées et ne sont jamais
|
||||
touchées par la purge.
|
||||
|
||||
Règles de sécurité, non négociables :
|
||||
- Aucune fonction de ce module ne renvoie une valeur, ne la journalise,
|
||||
ni ne l'écrit ailleurs que dans le coffre (chiffrée, via le store).
|
||||
Les aperçus ne contiennent que des noms, des types et ``has_value``.
|
||||
- L'import ne supprime rien : il crée des secrets dans le coffre et
|
||||
laisse ``api_keys.json`` intact. La purge est une étape séparée, qui
|
||||
exige ``confirmer=True`` et ne supprime que les noms listés.
|
||||
- Un secret déjà présent dans le coffre n'est JAMAIS écrasé : il est
|
||||
signalé dans ``deja_existants`` et la valeur du coffre est conservée.
|
||||
|
||||
Limite connue, documentée dans ``docs/COFFRE-sauvegarde-restauration.md`` :
|
||||
les lecteurs actuels (``backend/ai.py``, ``backend/tools/secrets.py``)
|
||||
lisent encore ``api_keys.json`` (chemin relatif ``data/``, sans passer
|
||||
par ``OBSIGATE_DATA_DIR``). Purger le clair sans avoir branché le
|
||||
lecteur concerné sur le coffre rend donc la clé inaccessible à ce
|
||||
lecteur : la purge reste un choix explicite de l'opérateur, confirmé
|
||||
deux fois côté interface (import, puis purge).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.coffre_store import (
|
||||
ValidationError,
|
||||
create_coffre,
|
||||
create_secret,
|
||||
get_coffre,
|
||||
get_secret,
|
||||
resolve_data_dir,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.coffre_migration")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
COFFRE_INTERNE_SLUG = "obsigate-interne"
|
||||
COFFRE_INTERNE_NOM = "ObsiGate interne"
|
||||
|
||||
# (nom dans api_keys.json, type de secret, service d'affichage).
|
||||
# Miroir des clés lues par backend/ai.py (clés IA) et par
|
||||
# backend/tools/secrets.py TOOL_KEY_NAMES (clés d'outils). L'URL Gitea
|
||||
# est volontairement absente : ce n'est pas un secret.
|
||||
CLES_MIGRABLES: tuple[tuple[str, str, str], ...] = (
|
||||
("DEEPSEEK_API_KEY", "api_key", "DeepSeek"),
|
||||
("OPENROUTER_API_KEY", "api_key", "OpenRouter"),
|
||||
("GEMINI_API_KEY", "api_key", "Gemini"),
|
||||
("NVIDIA_API_KEY", "api_key", "NVIDIA"),
|
||||
("QWENCLOUD_API_KEY", "api_key", "QwenCloud"),
|
||||
("XIAOMI_API_KEY", "api_key", "Xiaomi"),
|
||||
("MISTRAL_API_KEY", "api_key", "Mistral"),
|
||||
("OBSIGATE_TAVILY_API_KEY", "api_key", "Tavily"),
|
||||
("OBSIGATE_BRAVE_API_KEY", "api_key", "Brave"),
|
||||
("OBSIGATE_SERPAPI_API_KEY", "api_key", "SerpAPI"),
|
||||
("OBSIGATE_EXA_API_KEY", "api_key", "Exa"),
|
||||
("OBSIGATE_GITEA_TOKEN", "token", "Gitea"),
|
||||
("OBSIGATE_GITHUB_TOKEN", "token", "GitHub"),
|
||||
)
|
||||
|
||||
_NOMS_MIGRABLES = {nom for nom, _, _ in CLES_MIGRABLES}
|
||||
|
||||
|
||||
def _nom_secret(nom_source: str) -> str:
|
||||
"""Nom de secret dérivé du nom source (minuscules, valide NOM_SECRET_RE)."""
|
||||
return nom_source.lower()
|
||||
|
||||
|
||||
def _api_keys_path(base: Path) -> Path:
|
||||
return base / "api_keys.json"
|
||||
|
||||
|
||||
def _read_api_keys(base: Path) -> dict:
|
||||
path = _api_keys_path(base)
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("api_keys.json illisible (%s) : %s", path, exc)
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _write_api_keys(base: Path, data: dict) -> None:
|
||||
"""Écriture atomique tmp+rename, fichier final en 0600 (best-effort)."""
|
||||
path = _api_keys_path(base)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
try:
|
||||
os.chmod(tmp, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, path)
|
||||
try:
|
||||
os.chmod(path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def lister_candidats(data_dir: str | Path | None = None) -> list:
|
||||
"""Aperçu de migration : métadonnées seules, jamais de valeur.
|
||||
|
||||
Chaque candidat porte ``nom`` (nom dans le coffre), ``source`` (nom
|
||||
dans api_keys.json), ``type``, ``service``, ``has_value`` (une valeur
|
||||
non vide est présente dans le fichier) et ``deja_existant`` (un
|
||||
secret de ce nom existe déjà dans le coffre d'instance).
|
||||
"""
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
cles = _read_api_keys(base)
|
||||
coffre = get_coffre(COFFRE_INTERNE_SLUG, base)
|
||||
candidats = []
|
||||
for source, type_secret, service in CLES_MIGRABLES:
|
||||
nom = _nom_secret(source)
|
||||
valeur = cles.get(source)
|
||||
candidats.append(
|
||||
{
|
||||
"nom": nom,
|
||||
"source": source,
|
||||
"type": type_secret,
|
||||
"service": service,
|
||||
"has_value": isinstance(valeur, str) and bool(valeur.strip()),
|
||||
"deja_existant": (
|
||||
get_secret(coffre["id"], nom, base) is not None
|
||||
if coffre is not None
|
||||
else False
|
||||
),
|
||||
}
|
||||
)
|
||||
return candidats
|
||||
|
||||
|
||||
def importer_api_keys(
|
||||
data_dir: str | Path | None = None, cree_par: str = ""
|
||||
) -> dict:
|
||||
"""Importe les entrées avec valeur vers le coffre d'instance.
|
||||
|
||||
Ne supprime rien dans ``api_keys.json``. Retourne uniquement des
|
||||
noms et des comptes : ``migres``, ``deja_existants``, ``sans_valeur``.
|
||||
Lève ``KEKAbsenteError`` / ``CryptoIndisponibleError`` si le coffre
|
||||
ne peut pas chiffrer (le routeur les traduit en 503).
|
||||
"""
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
cles = _read_api_keys(base)
|
||||
coffre = get_coffre(COFFRE_INTERNE_SLUG, base)
|
||||
if coffre is None:
|
||||
coffre = create_coffre(
|
||||
slug=COFFRE_INTERNE_SLUG,
|
||||
nom=COFFRE_INTERNE_NOM,
|
||||
proprietaire=(cree_par or "admin").strip(),
|
||||
perimetre="instance",
|
||||
description="Coffre d'instance : clés IA et clés d'outils migrées depuis api_keys.json (Phase 2).",
|
||||
data_dir=base,
|
||||
)
|
||||
migres: list[str] = []
|
||||
deja_existants: list[str] = []
|
||||
sans_valeur: list[str] = []
|
||||
for source, type_secret, service in CLES_MIGRABLES:
|
||||
nom = _nom_secret(source)
|
||||
valeur = cles.get(source)
|
||||
if not (isinstance(valeur, str) and valeur.strip()):
|
||||
sans_valeur.append(nom)
|
||||
continue
|
||||
if get_secret(coffre["id"], nom, base) is not None:
|
||||
deja_existants.append(nom)
|
||||
continue
|
||||
create_secret(
|
||||
coffre,
|
||||
nom,
|
||||
type_secret,
|
||||
valeur,
|
||||
service=service,
|
||||
note="Migré depuis api_keys.json (Phase 2).",
|
||||
data_dir=base,
|
||||
)
|
||||
migres.append(nom)
|
||||
return {
|
||||
"coffre": COFFRE_INTERNE_SLUG,
|
||||
"migres": migres,
|
||||
"deja_existants": deja_existants,
|
||||
"sans_valeur": sans_valeur,
|
||||
}
|
||||
|
||||
|
||||
def purger_api_keys(
|
||||
noms: list[str] | None,
|
||||
data_dir: str | Path | None = None,
|
||||
confirmer: bool = False,
|
||||
) -> dict:
|
||||
"""Supprime du clair les noms listés, après confirmation explicite.
|
||||
|
||||
``confirmer`` doit valoir True, sinon refus (jamais automatique).
|
||||
Seuls les noms migrables sont acceptés ; un nom inconnu refuse
|
||||
l'opération entière, pour ne jamais supprimer une clé par erreur.
|
||||
Retourne ``supprimes`` et ``absents`` (noms, jamais de valeurs).
|
||||
"""
|
||||
if not confirmer:
|
||||
raise ValidationError(
|
||||
"Suppression du clair refusée : confirmation explicite requise"
|
||||
)
|
||||
demandes = [(n or "").strip().lower() for n in (noms or [])]
|
||||
# Accepte indifféremment le nom source (DEEPSEEK_API_KEY) ou le nom
|
||||
# de secret (deepseek_api_key) ; tout le reste refuse l'opération.
|
||||
noms_secrets = {_nom_secret(source) for source in _NOMS_MIGRABLES}
|
||||
inconnus = [
|
||||
n
|
||||
for n in demandes
|
||||
if n.upper() not in _NOMS_MIGRABLES and n not in noms_secrets
|
||||
]
|
||||
if inconnus:
|
||||
raise ValidationError(f"Noms inconnus, purge annulée : {', '.join(inconnus)}")
|
||||
sources_a_supprimer = set()
|
||||
for demande in demandes:
|
||||
for source, _, _ in CLES_MIGRABLES:
|
||||
if demande in (source.lower(), _nom_secret(source)):
|
||||
sources_a_supprimer.add(source)
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
cles = _read_api_keys(base)
|
||||
supprimes: list[str] = []
|
||||
absents: list[str] = []
|
||||
for source in sorted(sources_a_supprimer):
|
||||
if source in cles:
|
||||
del cles[source]
|
||||
supprimes.append(_nom_secret(source))
|
||||
else:
|
||||
absents.append(_nom_secret(source))
|
||||
_write_api_keys(base, cles)
|
||||
return {"supprimes": supprimes, "absents": absents}
|
||||
|
||||
|
||||
def creer_consommateur_avec_octrois(
|
||||
nom: str,
|
||||
octrois_demandes: list[dict[str, Any]] | None,
|
||||
cree_par: str,
|
||||
data_dir: str | Path | None = None,
|
||||
) -> tuple[dict, str, list, list]:
|
||||
"""Enregistre un consommateur et ses octrois en une seule opération.
|
||||
|
||||
Retourne (consommateur public, jeton à usage unique, octrois créés,
|
||||
erreurs). Une erreur sur un octroi (coffre ou secret introuvable)
|
||||
n'annule pas le consommateur ni les autres octrois : elle est
|
||||
signalée dans ``erreurs`` (noms seuls, jamais de valeurs). Le coffre
|
||||
lui-même n'est jamais modifié par cette opération.
|
||||
"""
|
||||
from backend.coffre_store import ( # import local : évite un cycle
|
||||
IntrouvableError,
|
||||
create_consommateur,
|
||||
create_octroi,
|
||||
)
|
||||
|
||||
base = resolve_data_dir(data_dir)
|
||||
consommateur, jeton = create_consommateur(nom, cree_par, base)
|
||||
crees: list[dict] = []
|
||||
erreurs: list[dict] = []
|
||||
for demande in octrois_demandes or []:
|
||||
slug = str(demande.get("coffre_slug", "") or "")
|
||||
secret_nom = str(demande.get("secret_nom", "") or "") or None
|
||||
try:
|
||||
coffre = get_coffre(slug, base)
|
||||
if coffre is None:
|
||||
raise IntrouvableError(f"Coffre introuvable : {slug}")
|
||||
secret_id = None
|
||||
if secret_nom is not None:
|
||||
secret = get_secret(coffre["id"], secret_nom, base)
|
||||
if secret is None:
|
||||
raise IntrouvableError(f"Secret introuvable : {secret_nom}")
|
||||
secret_id = secret["id"]
|
||||
octroi = create_octroi(
|
||||
coffre=coffre,
|
||||
cible_type="consommateur",
|
||||
cible_id=consommateur["id"],
|
||||
secret_id=secret_id,
|
||||
cree_par=cree_par,
|
||||
expiration=demande.get("expiration"),
|
||||
data_dir=base,
|
||||
)
|
||||
crees.append(octroi)
|
||||
except (IntrouvableError, ValidationError) as exc:
|
||||
erreurs.append(
|
||||
{
|
||||
"coffre_slug": slug,
|
||||
"secret_nom": secret_nom,
|
||||
"erreur": str(exc),
|
||||
}
|
||||
)
|
||||
return consommateur, jeton, crees, erreurs
|
||||
@@ -0,0 +1,860 @@
|
||||
"""Store du coffre Secrets ObsiGate (Phase 1a).
|
||||
|
||||
Patron repris de ``backend/tools/secrets.py`` : JSON dans ``data/``,
|
||||
écriture atomique (tmp + rename), verrou ``RLock`` sur les cycles
|
||||
lecture-modification-écriture, répertoire honorant ``OBSIGATE_DATA_DIR``.
|
||||
|
||||
Fichiers (permissions 0600) :
|
||||
- ``data/coffres.json`` : coffres (métadonnées + DEK chiffrées),
|
||||
consommateurs (jetons hachés SHA-256 seulement, jamais en clair),
|
||||
octrois (références, statuts, expirations).
|
||||
- ``data/coffres_valeurs.json`` : versions chiffrées des secrets.
|
||||
|
||||
Aucune valeur en clair n'est jamais persistée ici, et aucune fonction de
|
||||
ce module ne journalise de valeur ni de texte chiffré.
|
||||
|
||||
Modèle :
|
||||
- Coffre : id, slug, nom, description, proprietaire, perimetre
|
||||
(personnel | partage | instance), dek_chiffree (ou None en mode
|
||||
dégradé sans KEK), statut (actif | archive), dates.
|
||||
- Secret : id, coffre_id, nom, type, service, note, expiration,
|
||||
version_courante, cree_le, modifie_le, derniere_utilisation.
|
||||
- Version : numero, nonce, ciphertext, cree_le, remplacee_le.
|
||||
Seule la version courante est distribuable.
|
||||
- Consommateur : id (sert de jti), nom, jeton_hash, jeton_prefixe,
|
||||
statut (actif | revoque), cree_par, cree_le.
|
||||
- Octroi : id, coffre_id, secret_id (None = tout le coffre),
|
||||
cible_type (consommateur | utilisateur | executeur | workspace),
|
||||
cible_id, expiration, statut (actif | revoque | expire),
|
||||
cree_par, cree_le, revoque_le.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets as _secrets
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.coffre_crypto import (
|
||||
decrypt_secret_value,
|
||||
encrypt_secret_value,
|
||||
generate_dek,
|
||||
load_kek,
|
||||
parse_kek,
|
||||
reencrypt_secret_value,
|
||||
unwrap_dek,
|
||||
wrap_dek,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.coffre_store")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{2,63}$")
|
||||
NOM_SECRET_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{2,127}$")
|
||||
TYPES_SECRET = {"api_key", "token", "password", "certificate", "autre"}
|
||||
PERIMETRES = {"personnel", "partage", "instance"}
|
||||
CIBLES_OCTROI = {"consommateur", "utilisateur", "executeur", "workspace"}
|
||||
|
||||
MAX_VALEUR = 65536
|
||||
MAX_TEXTE = 2000
|
||||
DUREE_OCTROI_DEFAUT_JOURS = 90
|
||||
|
||||
# Motifs refusés dans les champs de métadonnées (description, note) :
|
||||
# une valeur ressemblant à un secret ne doit jamais transiter par un champ
|
||||
# qui, lui, est renvoyé en clair dans les métadonnées.
|
||||
_SECRET_LIKE_PATTERNS = (
|
||||
re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
||||
re.compile(r"\bsk-[A-Za-z0-9]{8,}\b"),
|
||||
re.compile(r"\bgh[opsu]_[A-Za-z0-9]{8,}\b"),
|
||||
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
||||
re.compile(
|
||||
r"(?i)\b(token|secret|mot_de_passe|password|api[-_]?key)\s*[:=]\s*\S+"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class StoreError(Exception):
|
||||
"""Erreur générique du store."""
|
||||
|
||||
|
||||
class CoffreExisteError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
class SecretExisteError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
class IntrouvableError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
class ValidationError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
class SecretExpireError(StoreError):
|
||||
pass
|
||||
|
||||
|
||||
def resolve_data_dir(data_dir: str | Path | None = None) -> Path:
|
||||
"""Répertoire de persistance (OBSIGATE_DATA_DIR, défaut ``data``)."""
|
||||
if data_dir is not None:
|
||||
return Path(data_dir)
|
||||
return Path(os.environ.get("OBSIGATE_DATA_DIR", "data"))
|
||||
|
||||
|
||||
def _coffres_path(base: Path) -> Path:
|
||||
return base / "coffres.json"
|
||||
|
||||
|
||||
def _valeurs_path(base: Path) -> Path:
|
||||
return base / "coffres_valeurs.json"
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> datetime | None:
|
||||
"""Parse une date ISO (datetime) ou une date seule (fin de journée UTC)."""
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
if len(text) == 10:
|
||||
text = text + "T23:59:59+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise ValidationError(f"Date invalide : {value!r}") from exc
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed
|
||||
|
||||
|
||||
def est_expire(expiration: str | None, maintenant: datetime | None = None) -> bool:
|
||||
"""True si la date d'expiration est passée (None = jamais)."""
|
||||
parsed = _parse_date(expiration)
|
||||
if parsed is None:
|
||||
return False
|
||||
ref = maintenant or datetime.now(UTC)
|
||||
return parsed <= ref
|
||||
|
||||
|
||||
def _read_json(path: Path, default: dict) -> dict:
|
||||
if not path.exists():
|
||||
return json.loads(json.dumps(default))
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("Store coffre illisible (%s) : %s", path, exc)
|
||||
return json.loads(json.dumps(default))
|
||||
return data if isinstance(data, dict) else json.loads(json.dumps(default))
|
||||
|
||||
|
||||
def _write_json(path: Path, data: dict) -> None:
|
||||
"""Écriture atomique tmp+rename, fichier final en 0600."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
try:
|
||||
os.chmod(tmp, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, path)
|
||||
try:
|
||||
os.chmod(path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_coffres(base: Path) -> dict:
|
||||
return _read_json(
|
||||
_coffres_path(base),
|
||||
{"coffres": [], "consommateurs": [], "octrois": []},
|
||||
)
|
||||
|
||||
|
||||
def _load_valeurs(base: Path) -> dict:
|
||||
return _read_json(_valeurs_path(base), {"versions": {}})
|
||||
|
||||
|
||||
def _save_coffres(base: Path, data: dict) -> None:
|
||||
_write_json(_coffres_path(base), data)
|
||||
|
||||
|
||||
def _save_valeurs(base: Path, data: dict) -> None:
|
||||
_write_json(_valeurs_path(base), data)
|
||||
|
||||
|
||||
def valider_slug(slug: str) -> str:
|
||||
value = (slug or "").strip().lower()
|
||||
if not SLUG_RE.match(value):
|
||||
raise ValidationError(
|
||||
"Slug invalide : 3 à 64 caractères, minuscules, chiffres, - ou _"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def valider_nom_secret(nom: str) -> str:
|
||||
value = (nom or "").strip().lower()
|
||||
if not NOM_SECRET_RE.match(value):
|
||||
raise ValidationError(
|
||||
"Nom de secret invalide : 3 à 128 caractères, minuscules, chiffres, - ou _"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def refuser_valeur_dans_metadonnees(texte: str | None, champ: str) -> None:
|
||||
"""Refuse une valeur ressemblant à un secret dans un champ métadonnée."""
|
||||
if not texte:
|
||||
return
|
||||
for motif in _SECRET_LIKE_PATTERNS:
|
||||
if motif.search(texte):
|
||||
raise ValidationError(
|
||||
f"Le champ {champ} ressemble à une valeur de secret ; "
|
||||
"utilisez le champ valeur du secret, jamais une métadonnée"
|
||||
)
|
||||
|
||||
|
||||
def _valider_texte(texte: str | None, champ: str) -> str:
|
||||
value = (texte or "").strip()
|
||||
if len(value) > MAX_TEXTE:
|
||||
raise ValidationError(f"Le champ {champ} dépasse {MAX_TEXTE} caractères")
|
||||
refuser_valeur_dans_metadonnees(value, champ)
|
||||
return value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coffres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def list_coffres(data_dir: str | Path | None = None) -> list:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
return list(_load_coffres(base)["coffres"])
|
||||
|
||||
|
||||
def get_coffre(slug: str, data_dir: str | Path | None = None) -> dict | None:
|
||||
base = resolve_data_dir(data_dir)
|
||||
slug = valider_slug(slug)
|
||||
with _lock:
|
||||
for coffre in _load_coffres(base)["coffres"]:
|
||||
if coffre["slug"] == slug:
|
||||
return coffre
|
||||
return None
|
||||
|
||||
|
||||
def get_coffre_or_404(slug: str, data_dir: str | Path | None = None) -> dict:
|
||||
coffre = get_coffre(slug, data_dir)
|
||||
if coffre is None:
|
||||
raise IntrouvableError(f"Coffre introuvable : {slug}")
|
||||
return coffre
|
||||
|
||||
|
||||
def get_coffre_by_id(coffre_id: str, data_dir: str | Path | None = None) -> dict | None:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
for coffre in _load_coffres(base)["coffres"]:
|
||||
if coffre["id"] == coffre_id:
|
||||
return coffre
|
||||
return None
|
||||
|
||||
|
||||
def get_octroi(octroi_id: str, data_dir: str | Path | None = None) -> dict | None:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
for octroi in _load_coffres(base)["octrois"]:
|
||||
if octroi["id"] == octroi_id:
|
||||
return octroi
|
||||
return None
|
||||
|
||||
|
||||
def create_coffre(
|
||||
slug: str,
|
||||
nom: str,
|
||||
proprietaire: str,
|
||||
perimetre: str = "personnel",
|
||||
description: str = "",
|
||||
data_dir: str | Path | None = None,
|
||||
) -> dict:
|
||||
"""Crée un coffre. Sans KEK, la DEK reste None (mode dégradé) : les
|
||||
métadonnées sont créées, mais aucun secret ne pourra être ajouté tant
|
||||
que la KEK n'est pas configurée."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
slug = valider_slug(slug)
|
||||
if perimetre not in PERIMETRES:
|
||||
raise ValidationError(f"Périmètre invalide : {perimetre}")
|
||||
if not (proprietaire or "").strip():
|
||||
raise ValidationError("Propriétaire requis")
|
||||
description = _valider_texte(description, "description")
|
||||
nom = (nom or "").strip()
|
||||
if not nom or len(nom) > 128:
|
||||
raise ValidationError("Nom de coffre requis (128 caractères max)")
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
if any(c["slug"] == slug for c in data["coffres"]):
|
||||
raise CoffreExisteError(f"Coffre déjà existant : {slug}")
|
||||
coffre: dict[str, Any] = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"slug": slug,
|
||||
"nom": nom,
|
||||
"description": description,
|
||||
"proprietaire": proprietaire.strip(),
|
||||
"perimetre": perimetre,
|
||||
"dek_chiffree": None,
|
||||
"statut": "actif",
|
||||
"cree_le": _now_iso(),
|
||||
}
|
||||
try:
|
||||
kek = load_kek()
|
||||
except Exception:
|
||||
kek = None
|
||||
if kek is not None:
|
||||
coffre["dek_chiffree"] = wrap_dek(generate_dek(), kek, coffre["id"])
|
||||
data["coffres"].append(coffre)
|
||||
_save_coffres(base, data)
|
||||
return coffre
|
||||
|
||||
|
||||
def ensure_dek(coffre: dict, data_dir: str | Path | None = None) -> bytes:
|
||||
"""Retourne la DEK en clair (mémoire seulement). Génère et persiste la
|
||||
DEK si le coffre a été créé en mode dégradé et que la KEK est là."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
kek = load_kek() # lève KEKAbsenteError si aucune KEK
|
||||
if coffre.get("dek_chiffree"):
|
||||
return unwrap_dek(coffre["dek_chiffree"], kek, coffre["id"])
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
for item in data["coffres"]:
|
||||
if item["id"] == coffre["id"]:
|
||||
if item.get("dek_chiffree"):
|
||||
return unwrap_dek(item["dek_chiffree"], kek, item["id"])
|
||||
dek = generate_dek()
|
||||
item["dek_chiffree"] = wrap_dek(dek, kek, item["id"])
|
||||
_save_coffres(base, data)
|
||||
return dek
|
||||
raise IntrouvableError("Coffre introuvable pour génération de DEK")
|
||||
|
||||
|
||||
# Les secrets vivent dans coffres.json (métadonnées), leurs versions dans
|
||||
# coffres_valeurs.json (chiffrées). Les deux chargements sont faits ici
|
||||
# pour garder une seule source de vérité dans ce module.
|
||||
def _load_tout(base: Path) -> tuple[dict, dict]:
|
||||
data = _load_coffres(base)
|
||||
if "secrets" not in data:
|
||||
data["secrets"] = []
|
||||
return data, _load_valeurs(base)
|
||||
|
||||
|
||||
def compter_secrets(coffre_id: str, data_dir: str | Path | None = None) -> int:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
data, _ = _load_tout(base)
|
||||
return sum(1 for s in data["secrets"] if s["coffre_id"] == coffre_id)
|
||||
|
||||
|
||||
def list_secrets(coffre_id: str, data_dir: str | Path | None = None) -> list:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
data, _ = _load_tout(base)
|
||||
return [s for s in data["secrets"] if s["coffre_id"] == coffre_id]
|
||||
|
||||
|
||||
def get_secret(
|
||||
coffre_id: str, nom: str, data_dir: str | Path | None = None
|
||||
) -> dict | None:
|
||||
base = resolve_data_dir(data_dir)
|
||||
nom = valider_nom_secret(nom)
|
||||
with _lock:
|
||||
data, _ = _load_tout(base)
|
||||
for secret in data["secrets"]:
|
||||
if secret["coffre_id"] == coffre_id and secret["nom"] == nom:
|
||||
return secret
|
||||
return None
|
||||
|
||||
|
||||
def secret_metadata(secret: dict, octrois_actifs: int = 0) -> dict:
|
||||
"""Vue métadonnées seules d'un secret : jamais de valeur, jamais de
|
||||
texte chiffré, jamais de nonce."""
|
||||
return {
|
||||
"id": secret["id"],
|
||||
"nom": secret["nom"],
|
||||
"type": secret["type"],
|
||||
"service": secret.get("service", ""),
|
||||
"note": secret.get("note", ""),
|
||||
"expiration": secret.get("expiration"),
|
||||
"expire": est_expire(secret.get("expiration")),
|
||||
"version": secret["version_courante"],
|
||||
"has_value": True,
|
||||
"cree_le": secret["cree_le"],
|
||||
"modifie_le": secret["modifie_le"],
|
||||
"derniere_utilisation": secret.get("derniere_utilisation"),
|
||||
"octrois_actifs": octrois_actifs,
|
||||
"reference": f"coffre:{secret['coffre_slug']}/{secret['nom']}",
|
||||
}
|
||||
|
||||
|
||||
def create_secret(
|
||||
coffre: dict,
|
||||
nom: str,
|
||||
type_secret: str,
|
||||
valeur: str,
|
||||
service: str = "",
|
||||
expiration: str | None = None,
|
||||
note: str = "",
|
||||
data_dir: str | Path | None = None,
|
||||
) -> dict:
|
||||
"""Chiffre et stocke un secret (nouvelle version 1)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
nom = valider_nom_secret(nom)
|
||||
if type_secret not in TYPES_SECRET:
|
||||
raise ValidationError(f"Type de secret invalide : {type_secret}")
|
||||
if not valeur or len(valeur) > MAX_VALEUR:
|
||||
raise ValidationError("Valeur requise (64 KiB max)")
|
||||
service = (service or "").strip()
|
||||
if len(service) > 128:
|
||||
raise ValidationError("Service trop long (128 caractères max)")
|
||||
note = _valider_texte(note, "note")
|
||||
if expiration is not None:
|
||||
_parse_date(expiration)
|
||||
dek = ensure_dek(coffre, base) # KEKAbsenteError si pas de KEK
|
||||
with _lock:
|
||||
data, valeurs = _load_tout(base)
|
||||
if any(
|
||||
s["coffre_id"] == coffre["id"] and s["nom"] == nom
|
||||
for s in data["secrets"]
|
||||
):
|
||||
raise SecretExisteError(f"Secret déjà existant : {nom}")
|
||||
secret_id = str(uuid.uuid4())
|
||||
chiffre = encrypt_secret_value(dek, secret_id, 1, valeur)
|
||||
secret = {
|
||||
"id": secret_id,
|
||||
"coffre_id": coffre["id"],
|
||||
"coffre_slug": coffre["slug"],
|
||||
"nom": nom,
|
||||
"type": type_secret,
|
||||
"service": service,
|
||||
"note": note,
|
||||
"expiration": expiration,
|
||||
"version_courante": 1,
|
||||
"cree_le": _now_iso(),
|
||||
"modifie_le": _now_iso(),
|
||||
"derniere_utilisation": None,
|
||||
}
|
||||
data["secrets"].append(secret)
|
||||
valeurs["versions"][secret_id] = [
|
||||
{
|
||||
"numero": 1,
|
||||
"nonce": chiffre["nonce"],
|
||||
"ciphertext": chiffre["ciphertext"],
|
||||
"cree_le": secret["cree_le"],
|
||||
"remplacee_le": None,
|
||||
}
|
||||
]
|
||||
_save_coffres(base, data)
|
||||
_save_valeurs(base, valeurs)
|
||||
return secret
|
||||
|
||||
|
||||
def rotate_secret(
|
||||
coffre: dict, nom: str, valeur: str, data_dir: str | Path | None = None
|
||||
) -> dict:
|
||||
"""Rotation : nouvelle version active immédiatement, l'ancienne n'est
|
||||
plus distribuable (seule la version courante est lue à la distribution)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
nom = valider_nom_secret(nom)
|
||||
if not valeur or len(valeur) > MAX_VALEUR:
|
||||
raise ValidationError("Valeur requise (64 KiB max)")
|
||||
dek = ensure_dek(coffre, base)
|
||||
with _lock:
|
||||
data, valeurs = _load_tout(base)
|
||||
secret = None
|
||||
for item in data["secrets"]:
|
||||
if item["coffre_id"] == coffre["id"] and item["nom"] == nom:
|
||||
secret = item
|
||||
break
|
||||
if secret is None:
|
||||
raise IntrouvableError(f"Secret introuvable : {nom}")
|
||||
nouvelle_version = secret["version_courante"] + 1
|
||||
chiffre = encrypt_secret_value(dek, secret["id"], nouvelle_version, valeur)
|
||||
versions = valeurs["versions"].setdefault(secret["id"], [])
|
||||
for version in versions:
|
||||
if version["remplacee_le"] is None:
|
||||
version["remplacee_le"] = _now_iso()
|
||||
versions.append(
|
||||
{
|
||||
"numero": nouvelle_version,
|
||||
"nonce": chiffre["nonce"],
|
||||
"ciphertext": chiffre["ciphertext"],
|
||||
"cree_le": _now_iso(),
|
||||
"remplacee_le": None,
|
||||
}
|
||||
)
|
||||
secret["version_courante"] = nouvelle_version
|
||||
secret["modifie_le"] = _now_iso()
|
||||
_save_coffres(base, data)
|
||||
_save_valeurs(base, valeurs)
|
||||
return secret
|
||||
|
||||
|
||||
def delete_secret(
|
||||
coffre: dict, nom: str, data_dir: str | Path | None = None
|
||||
) -> tuple[dict, list]:
|
||||
"""Supprime un secret, ses versions et ses octrois (cascade)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
nom = valider_nom_secret(nom)
|
||||
with _lock:
|
||||
data, valeurs = _load_tout(base)
|
||||
secret = None
|
||||
for item in data["secrets"]:
|
||||
if item["coffre_id"] == coffre["id"] and item["nom"] == nom:
|
||||
secret = item
|
||||
break
|
||||
if secret is None:
|
||||
raise IntrouvableError(f"Secret introuvable : {nom}")
|
||||
data["secrets"] = [s for s in data["secrets"] if s["id"] != secret["id"]]
|
||||
valeurs["versions"].pop(secret["id"], None)
|
||||
octrois_supprimes = [
|
||||
o for o in data["octrois"] if o.get("secret_id") == secret["id"]
|
||||
]
|
||||
data["octrois"] = [
|
||||
o for o in data["octrois"] if o.get("secret_id") != secret["id"]
|
||||
]
|
||||
_save_coffres(base, data)
|
||||
_save_valeurs(base, valeurs)
|
||||
return secret, octrois_supprimes
|
||||
|
||||
|
||||
def distribuer_valeur(
|
||||
coffre: dict, secret: dict, data_dir: str | Path | None = None
|
||||
) -> str:
|
||||
"""Déchiffre la version courante d'un secret (après contrôles d'octroi
|
||||
faits par le routeur). Refuse un secret expiré (410 côté routeur)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
if est_expire(secret.get("expiration")):
|
||||
raise SecretExpireError("Secret expiré")
|
||||
dek = ensure_dek(coffre, base)
|
||||
with _lock:
|
||||
_, valeurs = _load_tout(base)
|
||||
versions = valeurs["versions"].get(secret["id"], [])
|
||||
courante = None
|
||||
for version in versions:
|
||||
if version["numero"] == secret["version_courante"]:
|
||||
courante = version
|
||||
break
|
||||
if courante is None:
|
||||
raise IntrouvableError("Version courante introuvable")
|
||||
return decrypt_secret_value(
|
||||
dek,
|
||||
secret["id"],
|
||||
secret["version_courante"],
|
||||
courante["nonce"],
|
||||
courante["ciphertext"],
|
||||
)
|
||||
|
||||
|
||||
def toucher_utilisation(secret_id: str, data_dir: str | Path | None = None) -> None:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
data, _ = _load_tout(base)
|
||||
for secret in data["secrets"]:
|
||||
if secret["id"] == secret_id:
|
||||
secret["derniere_utilisation"] = _now_iso()
|
||||
_save_coffres(base, data)
|
||||
return
|
||||
|
||||
|
||||
def rotate_dek(coffre_slug: str, data_dir: str | Path | None = None) -> dict:
|
||||
"""Rotation DEK : re-chiffre toutes les versions du coffre avec une
|
||||
nouvelle DEK, puis remplace la DEK chiffrée du coffre."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
coffre = get_coffre_or_404(coffre_slug, base)
|
||||
kek = load_kek()
|
||||
ancienne_dek = ensure_dek(coffre, base)
|
||||
nouvelle_dek = generate_dek()
|
||||
with _lock:
|
||||
data, valeurs = _load_tout(base)
|
||||
secrets_coffre = [s for s in data["secrets"] if s["coffre_id"] == coffre["id"]]
|
||||
for secret in secrets_coffre:
|
||||
nouvelles_versions = []
|
||||
for version in valeurs["versions"].get(secret["id"], []):
|
||||
nouvelles_versions.append(
|
||||
{
|
||||
**reencrypt_secret_value(
|
||||
ancienne_dek,
|
||||
nouvelle_dek,
|
||||
secret["id"],
|
||||
version["numero"],
|
||||
version["nonce"],
|
||||
version["ciphertext"],
|
||||
),
|
||||
"numero": version["numero"],
|
||||
"cree_le": version["cree_le"],
|
||||
"remplacee_le": version["remplacee_le"],
|
||||
}
|
||||
)
|
||||
valeurs["versions"][secret["id"]] = nouvelles_versions
|
||||
for item in data["coffres"]:
|
||||
if item["id"] == coffre["id"]:
|
||||
item["dek_chiffree"] = wrap_dek(nouvelle_dek, kek, item["id"])
|
||||
coffre = item
|
||||
break
|
||||
_save_coffres(base, data)
|
||||
_save_valeurs(base, valeurs)
|
||||
return coffre
|
||||
|
||||
|
||||
def rotate_kek(nouvelle_kek_b64: str, data_dir: str | Path | None = None) -> int:
|
||||
"""Rotation KEK : re-chiffre uniquement les DEK avec la nouvelle KEK.
|
||||
L'opérateur doit ensuite remplacer OBSIGATE_KEK / OBSIGATE_KEK_FILE
|
||||
par la nouvelle valeur (cette fonction ne touche pas à l'environnement).
|
||||
Retourne le nombre de coffres re-chiffrés."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
ancienne_kek = load_kek()
|
||||
nouvelle_kek = parse_kek(nouvelle_kek_b64)
|
||||
count = 0
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
for coffre in data["coffres"]:
|
||||
if not coffre.get("dek_chiffree"):
|
||||
continue
|
||||
dek = unwrap_dek(coffre["dek_chiffree"], ancienne_kek, coffre["id"])
|
||||
coffre["dek_chiffree"] = wrap_dek(dek, nouvelle_kek, coffre["id"])
|
||||
count += 1
|
||||
_save_coffres(base, data)
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Consommateurs (jetons serveur-à-serveur, hachés, révocables par jti)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _hash_jeton(jeton: str) -> str:
|
||||
return hashlib.sha256(jeton.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def list_consommateurs(data_dir: str | Path | None = None) -> list:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
return [
|
||||
{k: v for k, v in c.items() if k != "jeton_hash"}
|
||||
for c in _load_coffres(base)["consommateurs"]
|
||||
]
|
||||
|
||||
|
||||
def create_consommateur(
|
||||
nom: str, cree_par: str, data_dir: str | Path | None = None
|
||||
) -> tuple[dict, str]:
|
||||
"""Crée un consommateur et retourne (métadonnées, jeton à usage unique).
|
||||
Le jeton n'est JAMAIS persisté : seul son hash SHA-256 est stocké."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
nom = (nom or "").strip()
|
||||
if not nom or len(nom) > 128:
|
||||
raise ValidationError("Nom de consommateur requis (128 caractères max)")
|
||||
jeton = "csk_" + _secrets.token_urlsafe(32)
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
consommateur = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"nom": nom,
|
||||
"jeton_hash": _hash_jeton(jeton),
|
||||
"jeton_prefixe": jeton[:12],
|
||||
"statut": "actif",
|
||||
"cree_par": cree_par,
|
||||
"cree_le": _now_iso(),
|
||||
}
|
||||
data["consommateurs"].append(consommateur)
|
||||
_save_coffres(base, data)
|
||||
public = {k: v for k, v in consommateur.items() if k != "jeton_hash"}
|
||||
return public, jeton
|
||||
|
||||
|
||||
def verifier_jeton_consommateur(
|
||||
jeton: str, data_dir: str | Path | None = None
|
||||
) -> dict | None:
|
||||
"""Retourne le consommateur (sans hash) si le jeton correspond, sinon None."""
|
||||
if not jeton:
|
||||
return None
|
||||
base = resolve_data_dir(data_dir)
|
||||
cible = _hash_jeton(jeton)
|
||||
with _lock:
|
||||
for consommateur in _load_coffres(base)["consommateurs"]:
|
||||
if hmac.compare_digest(consommateur.get("jeton_hash", ""), cible):
|
||||
return {
|
||||
k: v for k, v in consommateur.items() if k != "jeton_hash"
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def revoquer_consommateur(
|
||||
consommateur_id: str, data_dir: str | Path | None = None
|
||||
) -> dict:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
for consommateur in data["consommateurs"]:
|
||||
if consommateur["id"] == consommateur_id:
|
||||
consommateur["statut"] = "revoque"
|
||||
_save_coffres(base, data)
|
||||
return {
|
||||
k: v
|
||||
for k, v in consommateur.items()
|
||||
if k != "jeton_hash"
|
||||
}
|
||||
raise IntrouvableError("Consommateur introuvable")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Octrois
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def list_octrois(coffre_id: str, data_dir: str | Path | None = None) -> list:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
return [
|
||||
o for o in _load_coffres(base)["octrois"] if o["coffre_id"] == coffre_id
|
||||
]
|
||||
|
||||
|
||||
def compter_octrois_actifs(
|
||||
secret_id: str, data_dir: str | Path | None = None
|
||||
) -> int:
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
return sum(
|
||||
1
|
||||
for o in _load_coffres(base)["octrois"]
|
||||
if o.get("secret_id") == secret_id
|
||||
and o["statut"] == "actif"
|
||||
and not est_expire(o.get("expiration"))
|
||||
)
|
||||
|
||||
|
||||
def create_octroi(
|
||||
coffre: dict,
|
||||
cible_type: str,
|
||||
cible_id: str,
|
||||
secret_id: str | None,
|
||||
cree_par: str,
|
||||
expiration: str | None = None,
|
||||
data_dir: str | Path | None = None,
|
||||
) -> dict:
|
||||
base = resolve_data_dir(data_dir)
|
||||
if cible_type not in CIBLES_OCTROI:
|
||||
raise ValidationError(f"Type de cible invalide : {cible_type}")
|
||||
if not (cible_id or "").strip():
|
||||
raise ValidationError("Cible d'octroi requise")
|
||||
if secret_id is not None:
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
if not any(
|
||||
s["id"] == secret_id and s["coffre_id"] == coffre["id"]
|
||||
for s in data["secrets"]
|
||||
):
|
||||
raise IntrouvableError("Secret introuvable dans ce coffre")
|
||||
if expiration is None:
|
||||
expiration = (datetime.now(UTC) + timedelta(days=DUREE_OCTROI_DEFAUT_JOURS)).isoformat()
|
||||
else:
|
||||
_parse_date(expiration)
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
octroi = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"coffre_id": coffre["id"],
|
||||
"secret_id": secret_id,
|
||||
"cible_type": cible_type,
|
||||
"cible_id": cible_id.strip(),
|
||||
"expiration": expiration,
|
||||
"statut": "actif",
|
||||
"cree_par": cree_par,
|
||||
"cree_le": _now_iso(),
|
||||
"revoque_le": None,
|
||||
}
|
||||
data["octrois"].append(octroi)
|
||||
_save_coffres(base, data)
|
||||
return octroi
|
||||
|
||||
|
||||
def revoquer_octroi(octroi_id: str, data_dir: str | Path | None = None) -> dict:
|
||||
"""Révocation : statut -> revoque (conservé pour distinguer
|
||||
octroi_revoque de octroi_absent à la distribution)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
data = _load_coffres(base)
|
||||
for octroi in data["octrois"]:
|
||||
if octroi["id"] == octroi_id:
|
||||
octroi["statut"] = "revoque"
|
||||
octroi["revoque_le"] = _now_iso()
|
||||
_save_coffres(base, data)
|
||||
return octroi
|
||||
raise IntrouvableError("Octroi introuvable")
|
||||
|
||||
|
||||
def statut_octroi_consommateur(
|
||||
consommateur_id: str,
|
||||
coffre_id: str,
|
||||
secret_id: str,
|
||||
data_dir: str | Path | None = None,
|
||||
) -> tuple[str, dict | None]:
|
||||
"""Statut d'accès d'un consommateur à un secret.
|
||||
|
||||
Retourne ("ok" | "revoque" | "expire" | "absent", octroi). Un octroi
|
||||
coffre-wide (secret_id None) couvre tous les secrets du coffre.
|
||||
Priorité : ok > revoque > expire > absent (un octroi révoqué ne doit
|
||||
jamais être masqué par l'absence d'un autre octroi)."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
candidats = [
|
||||
o
|
||||
for o in _load_coffres(base)["octrois"]
|
||||
if o["cible_type"] == "consommateur"
|
||||
and o["cible_id"] == consommateur_id
|
||||
and o["coffre_id"] == coffre_id
|
||||
and (o.get("secret_id") is None or o["secret_id"] == secret_id)
|
||||
]
|
||||
for octroi in candidats:
|
||||
if octroi["statut"] == "actif" and not est_expire(octroi.get("expiration")):
|
||||
return "ok", octroi
|
||||
for octroi in candidats:
|
||||
if octroi["statut"] == "revoque":
|
||||
return "revoque", octroi
|
||||
for octroi in candidats:
|
||||
if octroi["statut"] == "actif" and est_expire(octroi.get("expiration")):
|
||||
return "expire", octroi
|
||||
return "absent", None
|
||||
|
||||
|
||||
def secrets_accessibles_utilisateur(
|
||||
username: str, coffre: dict, data_dir: str | Path | None = None
|
||||
) -> set | None:
|
||||
"""Secrets d'un coffre visibles par un utilisateur via octroi.
|
||||
|
||||
Retourne None si un octroi coffre-wide actif existe (tout le coffre),
|
||||
sinon l'ensemble des ids de secrets octroyés, sinon l'ensemble vide."""
|
||||
base = resolve_data_dir(data_dir)
|
||||
with _lock:
|
||||
octrois = [
|
||||
o
|
||||
for o in _load_coffres(base)["octrois"]
|
||||
if o["cible_type"] == "utilisateur"
|
||||
and o["cible_id"] == username
|
||||
and o["coffre_id"] == coffre["id"]
|
||||
and o["statut"] == "actif"
|
||||
and not est_expire(o.get("expiration"))
|
||||
]
|
||||
if any(o.get("secret_id") is None for o in octrois):
|
||||
return None
|
||||
return {o["secret_id"] for o in octrois if o.get("secret_id")}
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Collaboration temps réel — édition simultanée (ROADMAP #62).
|
||||
|
||||
Ce module implémente le cœur serveur de l'édition collaborative :
|
||||
|
||||
* un **relais WebSocket** : une *room* est créée par fichier ouvert
|
||||
(clé ``vault::chemin``) et tous les clients qui éditent le même fichier
|
||||
rejoignent la même room ;
|
||||
* un **relais de mises à jour Yjs** (CRDT) : le serveur ne décode pas le
|
||||
format binaire Yjs, il stocke le journal des mises à jour reçues et le
|
||||
rejoue aux nouveaux arrivants. La fusion sans conflit est assurée côté
|
||||
client par Yjs ;
|
||||
* un **awareness** (curseurs colorés + sélections) relayé entre clients ;
|
||||
* une **persistance différée** : le texte markdown reçu des clients est écrit
|
||||
sur disque après un debounce (2 s par défaut).
|
||||
|
||||
Sécurité : chaque connexion est authentifiée manuellement (les dépendances
|
||||
FastAPI ``Depends`` ne s'exécutent pas pour ``@app.websocket``), puis le
|
||||
chemin est validé via :func:`backend.services.paths.resolve_safe_path` et
|
||||
l'accès à la vault via ``check_vault_access``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import WebSocket
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
logger = logging.getLogger("obsigate.collab")
|
||||
|
||||
#: Délai (secondes) sans modification avant écriture sur disque.
|
||||
SAVE_DEBOUNCE_SECONDS = 2.0
|
||||
|
||||
#: Taille maximale d'une mise à jour Yjs encodée (protection anti-abus).
|
||||
MAX_UPDATE_BYTES = 8 * 1024 * 1024
|
||||
|
||||
#: Taille maximale d'un snapshot texte (protection anti-abus).
|
||||
MAX_TEXT_CHARS = 8 * 1024 * 1024
|
||||
|
||||
#: Taille maximale d'un message brut reçu (protection anti-abus, BUG-036).
|
||||
MAX_MESSAGE_CHARS = 16 * 1024 * 1024
|
||||
|
||||
#: Palette de couleurs attribuées aux utilisateurs (curseurs + avatars).
|
||||
PEER_COLORS = [
|
||||
"#e6194b", "#3cb44b", "#4363d8", "#f58231", "#911eb4",
|
||||
"#008080", "#9a6324", "#800000", "#808000", "#000075",
|
||||
]
|
||||
|
||||
|
||||
def color_for_index(index: int) -> str:
|
||||
"""Return a deterministic cursor color for a peer index."""
|
||||
return PEER_COLORS[index % len(PEER_COLORS)]
|
||||
|
||||
|
||||
def _b64encode(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii")
|
||||
|
||||
|
||||
def _b64decode(data: str) -> bytes:
|
||||
return base64.b64decode(data.encode("ascii"))
|
||||
|
||||
|
||||
def authenticate_websocket(websocket: WebSocket) -> dict[str, Any] | None:
|
||||
"""Authenticate a WebSocket connection.
|
||||
|
||||
Mirrors :func:`backend.auth.middleware.get_current_user` but works on the
|
||||
WebSocket scope: the JWT is read from the ``access_token`` cookie, which
|
||||
same-origin browsers send automatically during the handshake.
|
||||
|
||||
BUG-036: the token is **never** accepted from the query string anymore —
|
||||
URLs end up in access logs, proxies and browser history. Browsers cannot
|
||||
set custom headers on a WebSocket handshake, so the HttpOnly cookie set at
|
||||
login is the only supported transport.
|
||||
|
||||
Returns the user dict, or ``None`` if authentication fails.
|
||||
"""
|
||||
from backend.auth.jwt_handler import decode_token
|
||||
from backend.auth.middleware import is_auth_enabled
|
||||
from backend.auth.user_store import get_user
|
||||
|
||||
if not is_auth_enabled():
|
||||
return {
|
||||
"username": "anonymous",
|
||||
"display_name": "Anonymous",
|
||||
"role": "admin",
|
||||
"vaults": ["*"],
|
||||
"active": True,
|
||||
"_token_vaults": ["*"],
|
||||
}
|
||||
|
||||
token = websocket.cookies.get("access_token")
|
||||
if not token:
|
||||
return None
|
||||
|
||||
payload = decode_token(token)
|
||||
if not payload or payload.get("type") != "access":
|
||||
return None
|
||||
|
||||
user = get_user(payload["sub"])
|
||||
if not user or not user.get("active"):
|
||||
return None
|
||||
|
||||
user["_token_vaults"] = payload.get("vaults", [])
|
||||
user["_token_jti"] = payload.get("jti")
|
||||
return user
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollabClient:
|
||||
"""A single WebSocket connection inside a collaboration room."""
|
||||
|
||||
conn_id: int
|
||||
websocket: WebSocket
|
||||
username: str
|
||||
display_name: str
|
||||
color: str
|
||||
y_client_id: int | None = None
|
||||
awareness: dict[str, Any] | None = None
|
||||
|
||||
def peer(self) -> dict[str, Any]:
|
||||
return {
|
||||
"connId": self.conn_id,
|
||||
"clientId": self.y_client_id,
|
||||
"username": self.username,
|
||||
"displayName": self.display_name,
|
||||
"color": self.color,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollabRoom:
|
||||
"""State shared by every client editing the same file."""
|
||||
|
||||
vault: str
|
||||
path: str
|
||||
file_path: Path
|
||||
initial_text: str = ""
|
||||
clients: dict[int, CollabClient] = field(default_factory=dict)
|
||||
#: Journal des mises à jour Yjs (binaires) depuis la création de la room.
|
||||
updates: list[bytes] = field(default_factory=list)
|
||||
has_updates: bool = False
|
||||
seed_sent: bool = False
|
||||
pending_text: str | None = None
|
||||
save_task: asyncio.Task | None = None
|
||||
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return f"{self.vault}::{self.path}"
|
||||
|
||||
def peers(self) -> list[dict[str, Any]]:
|
||||
return [client.peer() for client in self.clients.values()]
|
||||
|
||||
def awareness_snapshot(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"clientId": c.y_client_id, "state": c.awareness}
|
||||
for c in self.clients.values()
|
||||
if c.y_client_id is not None and c.awareness is not None
|
||||
]
|
||||
|
||||
|
||||
class CollabManager:
|
||||
"""Manages collaboration rooms, broadcasting and disk persistence."""
|
||||
|
||||
def __init__(self, save_debounce: float = SAVE_DEBOUNCE_SECONDS) -> None:
|
||||
self._rooms: dict[str, CollabRoom] = {}
|
||||
self._save_debounce = save_debounce
|
||||
self._next_conn_id = 1
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# -- introspection (used by tests / diagnostics) ------------------------
|
||||
@property
|
||||
def room_count(self) -> int:
|
||||
return len(self._rooms)
|
||||
|
||||
def room_peer_count(self, vault: str, path: str) -> int:
|
||||
room = self._rooms.get(f"{vault}::{path}")
|
||||
return len(room.clients) if room else 0
|
||||
|
||||
def get_room(self, vault: str, path: str) -> CollabRoom | None:
|
||||
return self._rooms.get(f"{vault}::{path}")
|
||||
|
||||
# -- lifecycle ----------------------------------------------------------
|
||||
async def connect(
|
||||
self,
|
||||
websocket: WebSocket,
|
||||
vault: str,
|
||||
path: str,
|
||||
file_path: Path,
|
||||
user: dict[str, Any],
|
||||
) -> None:
|
||||
"""Register *websocket* in the room and relay messages until it closes."""
|
||||
async with self._lock:
|
||||
key = f"{vault}::{path}"
|
||||
room = self._rooms.get(key)
|
||||
if room is None:
|
||||
try:
|
||||
initial_text = file_path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
initial_text = ""
|
||||
room = CollabRoom(vault=vault, path=path, file_path=file_path, initial_text=initial_text)
|
||||
self._rooms[key] = room
|
||||
|
||||
conn_id = self._next_conn_id
|
||||
self._next_conn_id += 1
|
||||
client = CollabClient(
|
||||
conn_id=conn_id,
|
||||
websocket=websocket,
|
||||
username=user.get("username", "anonymous"),
|
||||
display_name=user.get("display_name") or user.get("username", "anonymous"),
|
||||
color=color_for_index(conn_id - 1),
|
||||
)
|
||||
room.clients[conn_id] = client
|
||||
|
||||
seed: str | None = None
|
||||
if not room.has_updates and not room.seed_sent:
|
||||
seed = room.initial_text
|
||||
room.seed_sent = True
|
||||
|
||||
await websocket.send_json({
|
||||
"type": "init",
|
||||
"connId": conn_id,
|
||||
"color": client.color,
|
||||
"seed": seed,
|
||||
"updates": [_b64encode(u) for u in room.updates],
|
||||
"peers": room.peers(),
|
||||
"awareness": room.awareness_snapshot(),
|
||||
})
|
||||
await self._broadcast(room, {"type": "peer_joined", "peer": client.peer()}, exclude=conn_id)
|
||||
|
||||
try:
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
await self._on_message(room, client, raw)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("Collab connection error (%s): %s", room.key, exc)
|
||||
finally:
|
||||
await self.disconnect(room, client)
|
||||
|
||||
async def disconnect(self, room: CollabRoom, client: CollabClient) -> None:
|
||||
"""Remove *client* from *room*, flushing and cleaning up if empty."""
|
||||
async with self._lock:
|
||||
room.clients.pop(client.conn_id, None)
|
||||
empty = not room.clients
|
||||
if empty:
|
||||
await self._flush(room)
|
||||
async with self._lock:
|
||||
# Only delete if nobody rejoined while we were flushing.
|
||||
if not room.clients and self._rooms.get(room.key) is room:
|
||||
if room.save_task:
|
||||
room.save_task.cancel()
|
||||
self._rooms.pop(room.key, None)
|
||||
else:
|
||||
await self._broadcast(
|
||||
room,
|
||||
{
|
||||
"type": "peer_left",
|
||||
"peer": client.peer(),
|
||||
"clientId": client.y_client_id,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Flush and cancel every room (called on application shutdown)."""
|
||||
async with self._lock:
|
||||
rooms = list(self._rooms.values())
|
||||
self._rooms.clear()
|
||||
for room in rooms:
|
||||
if room.save_task:
|
||||
room.save_task.cancel()
|
||||
await self._flush(room)
|
||||
|
||||
# -- message handling ---------------------------------------------------
|
||||
async def _on_message(self, room: CollabRoom, client: CollabClient, raw: str) -> None:
|
||||
# BUG-036: drop oversized frames before parsing them.
|
||||
if not isinstance(raw, str) or len(raw) > MAX_MESSAGE_CHARS:
|
||||
return
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return
|
||||
if not isinstance(message, dict):
|
||||
return
|
||||
|
||||
msg_type = message.get("type")
|
||||
|
||||
if msg_type in ("sync", "update"):
|
||||
encoded = message.get("update")
|
||||
if not isinstance(encoded, str):
|
||||
return
|
||||
try:
|
||||
update = _b64decode(encoded)
|
||||
except (ValueError, TypeError):
|
||||
return
|
||||
if not update or len(update) > MAX_UPDATE_BYTES:
|
||||
return
|
||||
async with self._lock:
|
||||
room.updates.append(update)
|
||||
room.has_updates = True
|
||||
await self._broadcast(
|
||||
room,
|
||||
{"type": "update", "update": encoded, "from": client.conn_id},
|
||||
exclude=client.conn_id,
|
||||
)
|
||||
|
||||
elif msg_type == "awareness":
|
||||
y_client_id = message.get("clientId")
|
||||
state = message.get("state")
|
||||
if not isinstance(y_client_id, int):
|
||||
return
|
||||
client.y_client_id = y_client_id
|
||||
client.awareness = state if isinstance(state, dict) else None
|
||||
await self._broadcast(
|
||||
room,
|
||||
{
|
||||
"type": "awareness",
|
||||
"clientId": y_client_id,
|
||||
"state": client.awareness,
|
||||
"from": client.conn_id,
|
||||
},
|
||||
exclude=client.conn_id,
|
||||
)
|
||||
|
||||
elif msg_type == "text":
|
||||
text = message.get("text")
|
||||
if not isinstance(text, str) or len(text) > MAX_TEXT_CHARS:
|
||||
return
|
||||
room.pending_text = text
|
||||
self._schedule_save(room)
|
||||
|
||||
elif msg_type == "ping":
|
||||
await client.websocket.send_json({"type": "pong", "t": int(time.time() * 1000)})
|
||||
|
||||
# -- broadcasting -------------------------------------------------------
|
||||
async def _broadcast(self, room: CollabRoom, message: dict[str, Any], exclude: int | None = None) -> None:
|
||||
dead: list[CollabClient] = []
|
||||
for client in list(room.clients.values()):
|
||||
if exclude is not None and client.conn_id == exclude:
|
||||
continue
|
||||
try:
|
||||
await client.websocket.send_json(message)
|
||||
except Exception:
|
||||
dead.append(client)
|
||||
for client in dead:
|
||||
room.clients.pop(client.conn_id, None)
|
||||
|
||||
# -- persistence --------------------------------------------------------
|
||||
def _schedule_save(self, room: CollabRoom) -> None:
|
||||
if room.save_task and not room.save_task.done():
|
||||
room.save_task.cancel()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError: # pragma: no cover - no running loop (tests)
|
||||
return
|
||||
room.save_task = loop.create_task(self._debounced_save(room))
|
||||
|
||||
async def _debounced_save(self, room: CollabRoom) -> None:
|
||||
try:
|
||||
await asyncio.sleep(self._save_debounce)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
await self._flush(room)
|
||||
|
||||
async def _flush(self, room: CollabRoom) -> None:
|
||||
"""Write the last received text snapshot to disk (if any)."""
|
||||
async with room.lock:
|
||||
text = room.pending_text
|
||||
room.pending_text = None
|
||||
if text is None:
|
||||
return
|
||||
try:
|
||||
await asyncio.to_thread(room.file_path.write_text, text, encoding="utf-8")
|
||||
logger.debug("Collab persisted %s", room.key)
|
||||
except OSError as exc:
|
||||
logger.warning("Collab persist failed for %s: %s", room.key, exc)
|
||||
|
||||
|
||||
#: Process-wide singleton used by the WebSocket endpoint.
|
||||
collab_manager = CollabManager()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Content-Security-Policy nonces (ROADMAP #87, tranche 5b).
|
||||
|
||||
Chaque réponse HTTP reçoit un nonce frais (``request.state.csp_nonce``)
|
||||
injecté dans ``script-src``. Les routes servant du HTML avec des scripts
|
||||
inline (index, popout, admin, editor-poc, excalidraw, page de partage)
|
||||
l'injectent dans le balisage via :func:`inject_csp_nonce` — mêmes
|
||||
emplacements, aucun script déplacé.
|
||||
|
||||
Tant que ``'unsafe-inline'`` reste dans la politique (retrait en T5c),
|
||||
l'injection est inerte : elle prépare la bascule sans changer le
|
||||
comportement.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import secrets
|
||||
|
||||
# Balises <script> exécutables sans `src` et sans nonce existant :
|
||||
# `<script>`, `<script type="module">`, `<script type="importmap">`.
|
||||
# Les blocs non-JS (ex. `type="text/plain"`) et les scripts externes
|
||||
# (`src=…`, couverts par 'self'/hôtes CDN) sont laissés intacts.
|
||||
_SCRIPT_TAG_RE = re.compile(
|
||||
r"<script(?=>|\s+type=\"(?:module|importmap)\"\s*>)",
|
||||
)
|
||||
|
||||
|
||||
def new_nonce() -> str:
|
||||
"""Generate a fresh per-response CSP nonce."""
|
||||
return secrets.token_urlsafe(16)
|
||||
|
||||
|
||||
def inject_csp_nonce(html: str, nonce: str) -> str:
|
||||
"""Add ``nonce="…"`` to bare executable inline ``<script>`` tags."""
|
||||
return _SCRIPT_TAG_RE.sub(f'<script nonce="{nonce}"', html)
|
||||
@@ -0,0 +1,207 @@
|
||||
"""URLs publiques de l'écosystème dracodev — par configuration (#223).
|
||||
|
||||
Règle dev/prod (Bruno, 2026-10-25) : **aucun nom public de
|
||||
l'écosystème n'est figé dans le code vivant**. Chaque URL publique
|
||||
vient de l'environnement ; les valeurs par défaut sont les noms
|
||||
**officiels** (sans suffixe) de la future production. L'environnement
|
||||
dev déclare ses noms en ``-dev`` dans son seul ``.env`` — le code
|
||||
n'en connaît aucun.
|
||||
|
||||
Variables (voir ``.env.example``) :
|
||||
|
||||
- ``DRACO_AGENT_URL`` — base publique du service d'agents ;
|
||||
à défaut, l'origine est **dérivée** de ``AGENT_EVENEMENTS_URL``
|
||||
(la configuration réelle de l'émission d'événements, #220), puis
|
||||
le défaut officiel.
|
||||
- ``DRACO_OBSIGATE_URL`` — base publique d'ObsiGate (liens profonds).
|
||||
- ``DRACO_FLOWDECK_URL`` — base publique de Flowdeck.
|
||||
- ``DRACO_TIKTAK_URL`` — base publique de TikTak.
|
||||
- ``DRACO_KANBANTE_URL`` — base publique de Kanbante.
|
||||
- ``DRACO_SHAARIT_URL`` — base publique de ShaarIt (barre v0.2.0).
|
||||
- ``DRACO_HUB_URL`` — base publique du Hub (barre v0.3.0,
|
||||
HUB-1 : liens Télécharger / À Propos / Aides / Profil du menu
|
||||
compte ; le Hub n'a pas d'onglet dans la barre).
|
||||
- ``DRACO_GITEA_URL`` — base publique de Gitea (les cartes
|
||||
Kanbante *sont* des issues Gitea, contrat liens-profonds) ;
|
||||
distincte de ``OBSIGATE_GITEA_URL`` (source connectée de
|
||||
l'assistant, réglable depuis l'UI, pas une URL d'affichage).
|
||||
|
||||
Résolution d'une URL sœur (BUG-120) : la variable explicite prime ;
|
||||
à défaut, l'**environnement est dérivé de l'hôte de l'agent** tel
|
||||
qu'ObsiGate le connaît déjà (:func:`agent_url`) — si cet hôte est de
|
||||
la forme ``agent<SUFFIXE>.dracodev.net`` (``-dev`` en dev, vide en
|
||||
prod), l'URL sœur est ``https://<slug><SUFFIXE>.dracodev.net``. Un
|
||||
hôte hors motif (autre domaine, LAN, IP, port) ne dérive **rien** :
|
||||
on ne fabrique jamais un lien deviné, le défaut officiel s'applique.
|
||||
L'installation pointe donc les bons liens selon son environnement
|
||||
sans déclarer les quatre ``DRACO_<APP>_URL``.
|
||||
|
||||
Ce module ne lit l'environnement qu'à l'appel (jamais à l'import) :
|
||||
les tests et un changement de ``.env`` suivi d'un redémarrage
|
||||
prennent effet sans état figé.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Défauts officiels (production future) — jamais de nom en ``-dev`` ici.
|
||||
DEFAUTS: dict[str, str] = {
|
||||
"agent": "https://agent.dracodev.net",
|
||||
"obsigate": "https://og.dracodev.net",
|
||||
"flowdeck": "https://flowdeck.dracodev.net",
|
||||
"tiktak": "https://tiktak.dracodev.net",
|
||||
"kanbante": "https://kanbante.dracodev.net",
|
||||
"shaarit": "https://shaarit.dracodev.net",
|
||||
"hub": "https://hub.dracodev.net",
|
||||
"gitea": "https://git.dracodev.net",
|
||||
}
|
||||
|
||||
_VARIABLES: dict[str, str] = {
|
||||
"agent": "DRACO_AGENT_URL",
|
||||
"obsigate": "DRACO_OBSIGATE_URL",
|
||||
"flowdeck": "DRACO_FLOWDECK_URL",
|
||||
"tiktak": "DRACO_TIKTAK_URL",
|
||||
"kanbante": "DRACO_KANBANTE_URL",
|
||||
"shaarit": "DRACO_SHAARIT_URL",
|
||||
"hub": "DRACO_HUB_URL",
|
||||
"gitea": "DRACO_GITEA_URL",
|
||||
}
|
||||
|
||||
# Slugs d'hôte des applications sœurs pour la dérivation
|
||||
# d'environnement (BUG-120) : premier label de ``<slug>.dracodev.net``
|
||||
# (et de ``<slug>-dev.dracodev.net`` en dev). Gitea est **exclue** :
|
||||
# son URL ne se dérive jamais.
|
||||
_SLUGS_HOTE: dict[str, str] = {
|
||||
"obsigate": "og",
|
||||
"flowdeck": "flowdeck",
|
||||
"tiktak": "tiktak",
|
||||
"kanbante": "kanbante",
|
||||
"shaarit": "shaarit",
|
||||
"hub": "hub",
|
||||
}
|
||||
|
||||
# Premier label de l'hôte de l'agent : ``agent`` suivi d'un suffixe
|
||||
# d'environnement éventuel (``-dev`` en dev, vide en prod).
|
||||
_MOTIF_AGENT = re.compile(r"^agent(-[a-z0-9-]*)?$")
|
||||
|
||||
|
||||
def _nettoyer(valeur: str) -> str:
|
||||
return valeur.strip().rstrip("/")
|
||||
|
||||
|
||||
def origine(url: str) -> str:
|
||||
"""Origine ``schéma://hôte[:port]`` d'une URL (``""`` si illisible)."""
|
||||
u = urlparse(url or "")
|
||||
if u.scheme and u.netloc:
|
||||
return f"{u.scheme}://{u.netloc}"
|
||||
return ""
|
||||
|
||||
|
||||
def agent_url() -> str:
|
||||
"""Base publique du service d'agents.
|
||||
|
||||
Priorité : ``DRACO_AGENT_URL``, puis l'origine de
|
||||
``AGENT_EVENEMENTS_URL`` (déjà la configuration réelle du poste),
|
||||
puis le défaut officiel.
|
||||
"""
|
||||
explicite = _nettoyer(os.environ.get(_VARIABLES["agent"], ""))
|
||||
if explicite:
|
||||
return explicite
|
||||
evenements = _nettoyer(os.environ.get("AGENT_EVENEMENTS_URL", ""))
|
||||
if evenements:
|
||||
derivee = origine(evenements)
|
||||
if derivee:
|
||||
return derivee
|
||||
return DEFAUTS["agent"]
|
||||
|
||||
|
||||
def agent_origine() -> str:
|
||||
"""Origine du service d'agents, pour la CSP (``connect-src``/``frame-src``)."""
|
||||
return origine(agent_url()) or DEFAUTS["agent"]
|
||||
|
||||
|
||||
def _suffixe_environnement() -> str | None:
|
||||
"""Suffixe d'environnement déduit de l'hôte de l'agent (BUG-120).
|
||||
|
||||
Lit l'hôte de :func:`agent_url` — l'origine de l'agent telle
|
||||
qu'ObsiGate la connaît déjà (``DRACO_AGENT_URL``, à défaut
|
||||
``AGENT_EVENEMENTS_URL``). Si cet hôte est exactement de la forme
|
||||
``agent<SUFFIXE>.dracodev.net`` sans port, retourne ``SUFFIXE``
|
||||
(``-dev`` en dev, ``""`` en prod). Sinon (autre domaine, LAN, IP,
|
||||
port exotique) retourne ``None`` : aucune dérivation, on ne
|
||||
fabrique jamais un lien deviné.
|
||||
"""
|
||||
u = urlparse(agent_url())
|
||||
hote = (u.hostname or "").lower()
|
||||
labels = hote.split(".")
|
||||
if len(labels) != 3 or labels[1:] != ["dracodev", "net"]:
|
||||
return None
|
||||
try:
|
||||
if u.port is not None:
|
||||
return None
|
||||
except ValueError:
|
||||
return None
|
||||
motif = _MOTIF_AGENT.match(labels[0])
|
||||
if motif is None:
|
||||
return None
|
||||
return motif.group(1) or ""
|
||||
|
||||
|
||||
def app_url(app: str) -> str:
|
||||
"""Base publique d'une application de l'écosystème (sans ``/`` final).
|
||||
|
||||
Résolution d'une URL sœur (BUG-120) : ``DRACO_<APP>_URL``
|
||||
explicite d'abord ; à défaut, dérivation d'environnement depuis
|
||||
l'hôte de l'agent (:func:`_suffixe_environnement`) ; à défaut,
|
||||
le défaut officiel. L'URL Gitea ne se dérive jamais.
|
||||
"""
|
||||
if app == "agent":
|
||||
return agent_url()
|
||||
variable = _VARIABLES[app]
|
||||
explicite = _nettoyer(os.environ.get(variable, ""))
|
||||
if explicite:
|
||||
return explicite
|
||||
if app in _SLUGS_HOTE:
|
||||
suffixe = _suffixe_environnement()
|
||||
if suffixe is not None:
|
||||
return f"https://{_SLUGS_HOTE[app]}{suffixe}.dracodev.net"
|
||||
return DEFAUTS[app]
|
||||
|
||||
|
||||
def app_hote(app: str) -> str:
|
||||
"""Nom d'hôte (minuscules) de l'URL publique configurée de *app*."""
|
||||
return (urlparse(app_url(app)).hostname or "").lower()
|
||||
|
||||
|
||||
def hotes_reconnus() -> dict[str, str]:
|
||||
"""``{hôte: app}`` pour le parseur de liens profonds.
|
||||
|
||||
Contient **toujours** les hôtes officiels (un lien collé depuis la
|
||||
production reste reconnaissable en dev, et réciproquement) **plus**
|
||||
l'hôte résolu de chaque application (les noms en ``-dev`` du poste,
|
||||
déclarés dans son ``.env`` ou dérivés de l'hôte de l'agent).
|
||||
"""
|
||||
hotes: dict[str, str] = {}
|
||||
for app, defaut in DEFAUTS.items():
|
||||
hote = (urlparse(defaut).hostname or "").lower()
|
||||
if hote:
|
||||
hotes[hote] = app
|
||||
for app in DEFAUTS:
|
||||
hote = app_hote(app)
|
||||
if hote:
|
||||
hotes[hote] = app
|
||||
return hotes
|
||||
|
||||
|
||||
def urls_publiques() -> dict[str, str]:
|
||||
"""Carte des URL publiques injectée à l'hôte web (``window.DRACO_URLS``).
|
||||
|
||||
C'est le contrat d'injection de dracodev-commons : l'application
|
||||
hôte (ici ObsiGate, qui sert ``index.html``) fournit les URL aux
|
||||
assets partagés (panneau Spark, barre, liens) — les assets ne
|
||||
portent que les défauts officiels en repli.
|
||||
"""
|
||||
return {app: app_url(app) for app in DEFAUTS}
|
||||
@@ -0,0 +1,463 @@
|
||||
# backend/draco_session.py
|
||||
# Fusion ACC-3.1 (#216) — Émettrice SSO « draco_session » (option 2a, C2).
|
||||
#
|
||||
# ObsiGate est l'unique émettrice du cookie de session partagé
|
||||
# ``draco_session`` (Domain=.dracodev.net) que les autres applications
|
||||
# de l'écosystème (Flowdeck, TikTak, Kanbante) valideront localement
|
||||
# (ACC-3.2, leurs propres tickets). Design : architecture Fusion §5.2 —
|
||||
# JWS compact HS256, claims minimaux (sub, name, iat, exp 12 h, jti),
|
||||
# aucun rôle dans le jeton, renouvellement glissant par l'émettrice,
|
||||
# révocation par jti exposée aux autres apps.
|
||||
#
|
||||
# Configuration (environnement) :
|
||||
# DRACO_SESSION_SECRET secret de signature partagé (obligatoire
|
||||
# pour activer la fonction ; sans lui,
|
||||
# aucun cookie n'est émis et rien ne change)
|
||||
# DRACO_SESSION_SECRET_PREV ancien secret accepté en vérification
|
||||
# pendant la fenêtre de rotation (24 h)
|
||||
# DRACO_SESSION_COOKIE_DOMAIN domaine du cookie (défaut .dracodev.net ;
|
||||
# vide = cookie lié à l'hôte, utile en dev)
|
||||
# DRACO_SSO_MACHINE_KEY clé machine protégeant la lecture de la
|
||||
# liste de révocation (GET /api/sso/revocations)
|
||||
#
|
||||
# Registre des émissions (HUB-13, fusion #127) : chaque jti émis est
|
||||
# consigné par `sub` dans data/draco_issued.json (jti -> exp, purgé à
|
||||
# l'expiration) pour que l'administration des comptes puisse révoquer
|
||||
# toutes les sessions draco d'un utilisateur (désactivation,
|
||||
# suppression, réinitialisation du mot de passe) via revoke().
|
||||
# HUB-22 (#227) ajoute la lecture de ce registre
|
||||
# (issued_sessions_for_user) et la révocation unitaire contrôlée par
|
||||
# appartenance (revoke_one_for_user), exposées par l'API machine des
|
||||
# sessions (backend/routers/comptes_machine.py).
|
||||
#
|
||||
# Rôles du Hub (HUB-19, #228) : à l'ouverture de session draco,
|
||||
# `user_for_draco` consulte en plus le registre des rôles du Hub
|
||||
# (`backend/hub_roles.py`, contrat D3(a) posé par le pilote ShaarIt) :
|
||||
# niveau connu ``aucun`` → la session draco est refusée (l'authenti-
|
||||
# fication native d'ObsiGate, émettrice, n'est jamais touchée) ;
|
||||
# niveau connu → il est joint au dict utilisateur (``_hub_role``)
|
||||
# pour que les routes d'administration du canal draco exigent
|
||||
# ``administrateur`` ; rôle inconnu (Hub injoignable) → comportement
|
||||
# actuel, aucun verrouillage nouveau. Aucun rôle dans le jeton :
|
||||
# validation locale, émission et révocations sont inchangées.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from jose import JWTError, jwt
|
||||
|
||||
logger = logging.getLogger("obsigate.draco_session")
|
||||
|
||||
COOKIE_NAME = "draco_session"
|
||||
ALGORITHM = "HS256"
|
||||
TOKEN_TTL_SECONDS = 12 * 3600 # durée de vie du jeton (design §5.2)
|
||||
RENEWAL_WINDOW_SECONDS = 2 * 3600 # réémettre si exp - now < 2 h
|
||||
REVOKED_FILE = Path("data/draco_revoked.json")
|
||||
ISSUED_FILE = Path("data/draco_issued.json")
|
||||
ALLOWED_RETURN_SUFFIX = "dracodev.net"
|
||||
|
||||
_lock = threading.Lock()
|
||||
_revoked: dict[str, int] | None = None # jti -> exp (chargé paresseusement)
|
||||
_issued: dict[str, dict[str, int]] | None = None # sub -> {jti -> exp}
|
||||
|
||||
|
||||
def _secret() -> str:
|
||||
return os.environ.get("DRACO_SESSION_SECRET", "")
|
||||
|
||||
|
||||
def _secret_prev() -> str:
|
||||
return os.environ.get("DRACO_SESSION_SECRET_PREV", "")
|
||||
|
||||
|
||||
def _cookie_domain() -> str:
|
||||
return os.environ.get("DRACO_SESSION_COOKIE_DOMAIN", ".dracodev.net")
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
"""La fonction est active seulement si le secret partagé est configuré."""
|
||||
return bool(_secret())
|
||||
|
||||
|
||||
# ── Révocation (jti) ────────────────────────────────────────────────────
|
||||
|
||||
def _load_revoked() -> dict[str, int]:
|
||||
global _revoked
|
||||
if _revoked is None:
|
||||
try:
|
||||
data = json.loads(REVOKED_FILE.read_text(encoding="utf-8"))
|
||||
_revoked = {str(k): int(v) for k, v in data.items()} \
|
||||
if isinstance(data, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
_revoked = {}
|
||||
return _revoked
|
||||
|
||||
|
||||
def _save_revoked() -> None:
|
||||
REVOKED_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = REVOKED_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(_revoked or {}), encoding="utf-8")
|
||||
tmp.replace(REVOKED_FILE)
|
||||
|
||||
|
||||
def revoke(jti: str, exp: int) -> None:
|
||||
"""Révoque un jeton par son jti (déconnexion globale)."""
|
||||
with _lock:
|
||||
store = _load_revoked()
|
||||
store[jti] = int(exp)
|
||||
now = int(time.time())
|
||||
for k in [k for k, v in store.items() if v <= now]:
|
||||
del store[k] # purge des expirés
|
||||
_save_revoked()
|
||||
|
||||
|
||||
def is_revoked(jti: str) -> bool:
|
||||
with _lock:
|
||||
return jti in _load_revoked()
|
||||
|
||||
|
||||
def revoked_list() -> list[dict]:
|
||||
"""Liste des révocations actives (pour les autres apps, cache 60 s)."""
|
||||
with _lock:
|
||||
now = int(time.time())
|
||||
return [{"jti": jti, "exp": exp}
|
||||
for jti, exp in _load_revoked().items() if exp > now]
|
||||
|
||||
|
||||
# ── Registre des émissions (par utilisateur) ────────────────────────────
|
||||
|
||||
def _load_issued() -> dict[str, dict[str, int]]:
|
||||
global _issued
|
||||
if _issued is None:
|
||||
try:
|
||||
data = json.loads(ISSUED_FILE.read_text(encoding="utf-8"))
|
||||
_issued = {
|
||||
str(sub): {str(j): int(e) for j, e in tokens.items()}
|
||||
for sub, tokens in data.items()
|
||||
if isinstance(tokens, dict)
|
||||
} if isinstance(data, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
_issued = {}
|
||||
return _issued
|
||||
|
||||
|
||||
def _save_issued() -> None:
|
||||
ISSUED_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = ISSUED_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(_issued or {}), encoding="utf-8")
|
||||
tmp.replace(ISSUED_FILE)
|
||||
|
||||
|
||||
def _register_issued(sub: str, jti: str, exp: int) -> None:
|
||||
"""Consigne un jti émis pour ``sub`` (purge des expirés au passage)."""
|
||||
if not sub:
|
||||
return
|
||||
try:
|
||||
with _lock:
|
||||
store = _load_issued()
|
||||
now = int(time.time())
|
||||
for key in list(store.keys()):
|
||||
store[key] = {j: e for j, e in store[key].items() if e > now}
|
||||
if not store[key]:
|
||||
del store[key]
|
||||
store.setdefault(sub, {})[jti] = int(exp)
|
||||
_save_issued()
|
||||
except Exception: # le registre ne casse jamais une émission
|
||||
logger.warning("Registre des émissions draco non écrit", exc_info=True)
|
||||
|
||||
|
||||
def revoke_all_for_user(sub: str) -> int:
|
||||
"""Révoque tous les jti draco actifs émis pour ``sub``.
|
||||
|
||||
Retourne le nombre de jti révoqués. Utilisé par l'administration
|
||||
des comptes (HUB-13) : désactivation, suppression, réinitialisation
|
||||
du mot de passe.
|
||||
"""
|
||||
if not sub:
|
||||
return 0
|
||||
with _lock:
|
||||
store = _load_issued()
|
||||
tokens = store.pop(str(sub), {})
|
||||
now = int(time.time())
|
||||
actifs = {j: e for j, e in tokens.items() if e > now}
|
||||
if tokens:
|
||||
_save_issued()
|
||||
for jti, exp in actifs.items():
|
||||
revoke(jti, exp)
|
||||
return len(actifs)
|
||||
|
||||
|
||||
def issued_sessions_for_user(sub: str) -> list[dict]:
|
||||
"""Sessions draco actives émises pour ``sub`` (HUB-22, #227).
|
||||
|
||||
Retourne ``[{jti, exp, revoquee}]`` triée par ``exp`` croissant,
|
||||
où ``revoquee`` indique si le ``jti`` figure déjà dans la liste des
|
||||
révocations. Les entrées expirées de cet utilisateur sont purgées
|
||||
du registre au passage (même patron que ``_register_issued``).
|
||||
"""
|
||||
if not sub:
|
||||
return []
|
||||
with _lock:
|
||||
store = _load_issued()
|
||||
now = int(time.time())
|
||||
tokens = store.get(str(sub), {})
|
||||
actifs = {j: e for j, e in tokens.items() if e > now}
|
||||
if len(actifs) != len(tokens):
|
||||
if actifs:
|
||||
store[str(sub)] = actifs
|
||||
else:
|
||||
store.pop(str(sub), None)
|
||||
_save_issued()
|
||||
revoques = _load_revoked()
|
||||
entrees = sorted(actifs.items(), key=lambda kv: kv[1])
|
||||
return [{"jti": jti, "exp": exp, "revoquee": jti in revoques}
|
||||
for jti, exp in entrees]
|
||||
|
||||
|
||||
def revoke_one_for_user(sub: str, jti: str) -> bool:
|
||||
"""Révoque un seul ``jti`` draco de ``sub`` (HUB-22, #227).
|
||||
|
||||
Ne révoque (via ``revoke``) que si le ``jti`` appartient bien à
|
||||
l'ensemble émis de ce ``sub`` et n'est pas expiré — jamais de
|
||||
révocation d'une session d'un autre utilisateur. Retourne ``True``
|
||||
si la révocation a eu lieu, ``False`` sinon.
|
||||
"""
|
||||
if not sub or not jti:
|
||||
return False
|
||||
with _lock:
|
||||
store = _load_issued()
|
||||
exp = store.get(str(sub), {}).get(jti)
|
||||
if exp is None or exp <= int(time.time()):
|
||||
return False
|
||||
revoke(jti, exp)
|
||||
return True
|
||||
|
||||
|
||||
# ── Émission / vérification ─────────────────────────────────────────────
|
||||
|
||||
def issue_token(user: dict) -> tuple[str, str, int]:
|
||||
"""Émet un JWS pour l'utilisateur. Retourne (jeton, jti, exp)."""
|
||||
now = int(time.time())
|
||||
exp = now + TOKEN_TTL_SECONDS
|
||||
jti = uuid.uuid4().hex
|
||||
claims = {
|
||||
"sub": str(user.get("id") or user.get("username") or ""),
|
||||
"name": str(user.get("display_name") or user.get("username") or ""),
|
||||
"iat": now,
|
||||
"exp": exp,
|
||||
"jti": jti,
|
||||
}
|
||||
token = jwt.encode(claims, _secret(), algorithm=ALGORITHM)
|
||||
_register_issued(str(claims["sub"]), jti, exp)
|
||||
return token, jti, exp
|
||||
|
||||
|
||||
def decode_claims(token: str) -> dict | None:
|
||||
"""Vérifie signature et expiration (pas la révocation)."""
|
||||
if not token or not enabled():
|
||||
return None
|
||||
for secret in (_secret(), _secret_prev()):
|
||||
if not secret:
|
||||
continue
|
||||
try:
|
||||
return jwt.decode(token, secret, algorithms=[ALGORITHM])
|
||||
except JWTError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def verify_token(token: str) -> dict | None:
|
||||
"""decode_claims + contrôle de révocation du jti."""
|
||||
claims = decode_claims(token)
|
||||
if claims is None:
|
||||
return None
|
||||
if claims.get("jti") and is_revoked(str(claims["jti"])):
|
||||
return None
|
||||
return claims
|
||||
|
||||
|
||||
def needs_renewal(claims: dict) -> bool:
|
||||
"""Vrai si le jeton entre dans la fenêtre de renouvellement (< 2 h)."""
|
||||
try:
|
||||
return int(claims["exp"]) - int(time.time()) < RENEWAL_WINDOW_SECONDS
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return True
|
||||
|
||||
|
||||
def set_cookie(response, user: dict, secure: bool) -> None:
|
||||
"""Pose le cookie draco_session sur une réponse (émission initiale)."""
|
||||
if not enabled():
|
||||
return
|
||||
token, _jti, _exp = issue_token(user)
|
||||
domain = _cookie_domain()
|
||||
extra = {"domain": domain} if domain else {}
|
||||
response.set_cookie(COOKIE_NAME, token, max_age=TOKEN_TTL_SECONDS,
|
||||
httponly=True, samesite="lax", secure=secure,
|
||||
path="/", **extra)
|
||||
|
||||
|
||||
def clear_cookie(response) -> None:
|
||||
domain = _cookie_domain()
|
||||
extra = {"domain": domain} if domain else {}
|
||||
response.delete_cookie(COOKIE_NAME, path="/", **extra)
|
||||
|
||||
|
||||
# ── Validation locale (ACC-3.2, #9) ─────────────────────────────────────
|
||||
|
||||
def user_for_draco(request) -> dict | None:
|
||||
"""Utilisateur ObsiGate prouvé par le cookie draco_session.
|
||||
|
||||
ObsiGate est à la fois émettrice et consommatrice : le `sub` du
|
||||
jeton EST son identifiant d'utilisateur — la correspondance est
|
||||
directe, par id. Le garde-fou « jeton antérieur au dernier
|
||||
changement de mot de passe » (BUG-028 côté natif) est appliqué
|
||||
à l'identique via le claim `iat`.
|
||||
"""
|
||||
claims = verify_token(request.cookies.get(COOKIE_NAME, ""))
|
||||
if not claims:
|
||||
return None
|
||||
sub = str(claims.get("sub", ""))
|
||||
if not sub:
|
||||
return None
|
||||
from backend.auth.user_store import get_all_users
|
||||
|
||||
for candidate in get_all_users():
|
||||
if str(candidate.get("id")) != sub:
|
||||
continue
|
||||
if not candidate.get("active"):
|
||||
return None
|
||||
pca = candidate.get("password_changed_at")
|
||||
iat = claims.get("iat")
|
||||
if pca is not None and iat is not None:
|
||||
try:
|
||||
if int(iat) < int(float(pca)):
|
||||
return None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
user = dict(candidate)
|
||||
user["_token_vaults"] = candidate.get("vaults", [])
|
||||
user["_token_scope"] = "full"
|
||||
user["_draco_sub"] = sub
|
||||
# HUB-19 (#228) : rôle du registre du Hub pour ce porteur.
|
||||
# Niveau connu « aucun » → pas de session établie par draco :
|
||||
# le refus est consigné sur la requête pour que `require_auth`
|
||||
# réponde un message explicite en français (et non un simple
|
||||
# 401 générique). Rôle connu → joint au dict (`_hub_role`) ;
|
||||
# rôle inconnu → `_hub_role` vaut None et le comportement
|
||||
# actuel s'applique, sans verrouillage nouveau.
|
||||
from backend import hub_roles
|
||||
|
||||
cookie = ""
|
||||
try:
|
||||
cookie = request.headers.get("cookie", "") or ""
|
||||
except Exception: # un objet requête minimal (tests) ne casse rien
|
||||
cookie = ""
|
||||
role = hub_roles.role_pour(sub, cookie)
|
||||
if role == "aucun":
|
||||
try:
|
||||
request.state.role_hub_refuse = "aucun"
|
||||
except Exception: # pas d'état mutable : refus quand même
|
||||
pass
|
||||
return None
|
||||
user["_hub_role"] = role
|
||||
return user
|
||||
return None
|
||||
|
||||
|
||||
# ── Renouvellement glissant (middleware) ────────────────────────────────
|
||||
|
||||
class DracoRenewalMiddleware:
|
||||
"""Réémet le cookie quand l'utilisateur est actif chez l'émettrice.
|
||||
|
||||
Middleware ASGI pur (pas de BaseHTTPMiddleware : la pile d'ObsiGate
|
||||
streame des réponses SSE/WS que BaseHTTPMiddleware mettrait en
|
||||
tampon). On enveloppe l'envoi de la réponse : si la requête porte
|
||||
un cookie d'accès natif valide et que le cookie draco est absent,
|
||||
invalide ou dans sa fenêtre de renouvellement (< 2 h), on ajoute
|
||||
un Set-Cookie frais aux en-têtes de la réponse. La révocation
|
||||
ciblée d'un jti draco est ainsi « guérie » par la prochaine
|
||||
activité authentifiée locale — la déconnexion globale, elle,
|
||||
révoque aussi les jetons natifs, donc plus rien n'est réémis.
|
||||
"""
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope.get("type") != "http" or not enabled():
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
cookie_header = ""
|
||||
for name, value in scope.get("headers") or []:
|
||||
if name == b"cookie":
|
||||
cookie_header = value.decode("latin-1")
|
||||
break
|
||||
cookies = {}
|
||||
for part in cookie_header.split(";"):
|
||||
if "=" in part:
|
||||
k, v = part.strip().split("=", 1)
|
||||
cookies[k] = v
|
||||
user = self._user_from_native(cookies.get("access_token", ""))
|
||||
if user is None:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
claims = verify_token(cookies.get(COOKIE_NAME, "")) \
|
||||
if cookies.get(COOKIE_NAME) else None
|
||||
if claims is not None and not needs_renewal(claims):
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
token, _jti, _exp = issue_token(user)
|
||||
domain = _cookie_domain()
|
||||
cookie = (f"{COOKIE_NAME}={token}; Max-Age={TOKEN_TTL_SECONDS}; "
|
||||
f"Path=/; HttpOnly; SameSite=lax")
|
||||
if domain:
|
||||
cookie += f"; Domain={domain}"
|
||||
if scope.get("scheme") == "https":
|
||||
cookie += "; Secure"
|
||||
|
||||
async def send_with_cookie(message):
|
||||
if message.get("type") == "http.response.start":
|
||||
headers = list(message.get("headers") or [])
|
||||
headers.append((b"set-cookie", cookie.encode("latin-1")))
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, receive, send_with_cookie)
|
||||
|
||||
@staticmethod
|
||||
def _user_from_native(access_token: str):
|
||||
"""Utilisateur local prouvé par le cookie d'accès natif, ou None."""
|
||||
if not access_token:
|
||||
return None
|
||||
try:
|
||||
from backend.auth.jwt_handler import decode_token
|
||||
from backend.auth.user_store import get_user
|
||||
payload = decode_token(access_token)
|
||||
if not payload or payload.get("type") != "access":
|
||||
return None
|
||||
return get_user(str(payload.get("sub", "")))
|
||||
except Exception: # le renouvellement ne casse jamais une requête
|
||||
return None
|
||||
|
||||
|
||||
# ── return_to (anti open-redirect) ──────────────────────────────────────
|
||||
|
||||
def validate_return_to(url: str) -> str | None:
|
||||
"""Accepte uniquement https vers dracodev.net ou un sous-domaine."""
|
||||
if not url:
|
||||
return None
|
||||
parsed = urlparse(url)
|
||||
host = (parsed.hostname or "").lower()
|
||||
if parsed.scheme != "https" or not host:
|
||||
return None
|
||||
if host == ALLOWED_RETURN_SUFFIX or host.endswith("." + ALLOWED_RETURN_SUFFIX):
|
||||
return url
|
||||
return None
|
||||
@@ -0,0 +1,434 @@
|
||||
"""
|
||||
Export utilities for ObsiGate — standalone HTML, Markdown bundle (.zip) and ePub.
|
||||
|
||||
Each function converts vault content (markdown notes) into a self-contained
|
||||
downloadable artifact. Rendering reuses ``mistune`` (already a core dependency)
|
||||
so no extra markdown engine is required. The ePub generator builds a minimal
|
||||
but valid EPUB 3 container by hand (``zipfile`` only) so there is no hard
|
||||
dependency on ``ebooklib``.
|
||||
|
||||
All functions raise :class:`ExportError` on any failure (missing file, binary
|
||||
content, unreadable source, ...) so callers can translate to HTTP errors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime
|
||||
import html as html_mod
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import unicodedata
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import frontmatter
|
||||
import mistune
|
||||
|
||||
logger = logging.getLogger("obsigate.export")
|
||||
|
||||
|
||||
class ExportError(Exception):
|
||||
"""Raised when an export operation cannot be completed."""
|
||||
|
||||
|
||||
# Cached mistune renderer (singleton) — mirrors backend.main's configuration.
|
||||
_markdown = mistune.create_markdown(
|
||||
escape=False,
|
||||
plugins=["table", "strikethrough", "footnotes", "task_lists"],
|
||||
)
|
||||
|
||||
# File extensions considered "safe" text files for markdown-like exports.
|
||||
_MARKDOWN_EXTS = {".md", ".markdown", ".mdown", ".mkd", ".txt"}
|
||||
|
||||
_HTML_CSS = """
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
max-width: 760px;
|
||||
margin: 40px auto;
|
||||
padding: 0 24px;
|
||||
line-height: 1.7;
|
||||
color: #1a1a2e;
|
||||
font-size: 16px;
|
||||
}
|
||||
h1 { font-size: 28px; border-bottom: 2px solid #333; padding-bottom: 8px; margin-top: 0; }
|
||||
h2 { font-size: 22px; margin-top: 28px; border-bottom: 1px solid #ddd; padding-bottom: 4px; }
|
||||
h3 { font-size: 18px; margin-top: 24px; }
|
||||
h4, h5, h6 { font-size: 16px; margin-top: 20px; }
|
||||
p { margin: 10px 0; }
|
||||
pre {
|
||||
background: #f5f5f5;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 6px;
|
||||
padding: 12px 16px;
|
||||
font-size: 14px;
|
||||
overflow-x: auto;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
code { font-size: 14px; background: #f5f5f5; padding: 2px 5px; border-radius: 3px; }
|
||||
pre code { background: none; padding: 0; }
|
||||
a { color: #4f46e5; text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
img { max-width: 100%; border-radius: 4px; }
|
||||
blockquote { border-left: 3px solid #ccc; padding-left: 14px; color: #555; margin: 12px 0; }
|
||||
table { border-collapse: collapse; width: 100%; margin: 12px 0; }
|
||||
th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; font-size: 14px; }
|
||||
th { background: #f0f0f0; font-weight: 600; }
|
||||
ul, ol { margin: 8px 0; padding-left: 24px; }
|
||||
li { margin: 2px 0; }
|
||||
hr { border: none; border-top: 1px solid #ddd; margin: 20px 0; }
|
||||
nav.export-nav {
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 8px;
|
||||
padding: 14px 18px;
|
||||
margin: 16px 0 28px;
|
||||
background: #fafafa;
|
||||
}
|
||||
nav.export-nav h2 { border: none; margin: 0 0 8px; font-size: 15px; text-transform: uppercase; letter-spacing: .04em; }
|
||||
nav.export-nav ul { margin: 0; padding-left: 20px; }
|
||||
.wikilink-missing { color: #c0392b; }
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""URL-safe slug from arbitrary text (Unicode-aware)."""
|
||||
text = html_mod.unescape(re.sub(r"<[^>]+>", "", text))
|
||||
text = text.lower()
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
||||
cleaned = []
|
||||
for ch in text:
|
||||
if unicodedata.category(ch).startswith(("L", "N")) or ch in (" ", "-"):
|
||||
cleaned.append(ch)
|
||||
text = re.sub(r"\s+", "-", "".join(cleaned))
|
||||
text = re.sub(r"-+", "-", text)
|
||||
return text.strip("-") or "document"
|
||||
|
||||
|
||||
def _resolve(vault_path: Path, file_path: Path | str) -> Path:
|
||||
"""Resolve *file_path* against *vault_path*, accepting absolute or relative.
|
||||
|
||||
Returns an absolute :class:`Path`. Does not require the file to exist.
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
if file_path.is_absolute():
|
||||
return file_path
|
||||
return (Path(vault_path) / file_path).resolve()
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
"""Read a text file, raising :class:`ExportError` on missing/binary content."""
|
||||
if not path.exists() or not path.is_file():
|
||||
raise ExportError(f"File not found: {path}")
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as e:
|
||||
raise ExportError(f"Cannot read file {path}: {e}") from e
|
||||
if b"\x00" in raw[:4096]:
|
||||
raise ExportError(f"File appears to be binary: {path}")
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _strip_frontmatter(text: str) -> tuple[str, dict]:
|
||||
"""Split frontmatter from a markdown document.
|
||||
|
||||
Returns ``(body, metadata)``. Falls back gracefully when no frontmatter.
|
||||
"""
|
||||
try:
|
||||
post = frontmatter.loads(text)
|
||||
return post.content, dict(post.metadata or {})
|
||||
except Exception:
|
||||
return text, {}
|
||||
|
||||
|
||||
def _safe_name(name: str) -> str:
|
||||
"""ASCII-safe, filename-safe download name."""
|
||||
cleaned = "".join(
|
||||
c for c in name if c.isascii() and (c.isalnum() or c in " _-.")
|
||||
).strip()
|
||||
return cleaned or "document"
|
||||
|
||||
|
||||
def _collect_markdown_files(vault_path: Path) -> list[Path]:
|
||||
"""List all markdown files in the vault, sorted by relative path."""
|
||||
vault_path = Path(vault_path)
|
||||
results: list[Path] = []
|
||||
if not vault_path.is_dir():
|
||||
return results
|
||||
for p in sorted(vault_path.rglob("*")):
|
||||
if p.is_file() and p.suffix.lower() in _MARKDOWN_EXTS:
|
||||
results.append(p)
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown → HTML (shared by HTML and ePub exporters)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _inline_images(md: str, file_dir: Path, vault_path: Path) -> str:
|
||||
"""Replace image references with base64 data URIs.
|
||||
|
||||
Supports standard markdown ```` and Obsidian embeds
|
||||
``![[image.png]]`` / ``![[image.png|300]]``. Sources are resolved relative
|
||||
to the note's directory, then the vault root.
|
||||
"""
|
||||
candidates = [file_dir, vault_path]
|
||||
|
||||
def _to_data_uri(src: str) -> str:
|
||||
src = src.strip().strip("<>")
|
||||
if src.startswith(("http://", "https://", "data:")):
|
||||
return src # leave remote/data URLs untouched
|
||||
src_path = None
|
||||
for base in candidates:
|
||||
candidate = (base / src).resolve()
|
||||
if candidate.is_file():
|
||||
src_path = candidate
|
||||
break
|
||||
if src_path is None:
|
||||
return src # unresolved — leave as-is
|
||||
try:
|
||||
data = src_path.read_bytes()
|
||||
mime = mimetypes.guess_type(str(src_path))[0] or "application/octet-stream"
|
||||
b64 = base64.b64encode(data).decode("ascii")
|
||||
return f"data:{mime};base64,{b64}"
|
||||
except OSError:
|
||||
return src
|
||||
|
||||
# Obsidian embeds: ![[path]] or ![[path|size]]
|
||||
def _embed(match):
|
||||
inner = match.group(1).strip()
|
||||
target = inner.split("|", 1)[0].strip()
|
||||
return f"})"
|
||||
|
||||
md = re.sub(r"!\[\[([^\]]+)\]\]", _embed, md)
|
||||
|
||||
# Standard markdown images
|
||||
def _img(match):
|
||||
alt = match.group(1)
|
||||
src = match.group(2)
|
||||
return f"})"
|
||||
|
||||
md = re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", _img, md)
|
||||
return md
|
||||
|
||||
|
||||
def _convert_wikilinks(md: str, vault_path: Path, current: Path) -> str:
|
||||
"""Convert ``[[Note]]`` / ``[[Note|display]]`` to cross-file HTML links."""
|
||||
md_files = {p.stem.lower(): p for p in _collect_markdown_files(vault_path)}
|
||||
|
||||
def _replace(match):
|
||||
target = match.group(1).strip()
|
||||
display = (match.group(2) or target).strip()
|
||||
|
||||
# Same-document anchor
|
||||
if target.startswith("#"):
|
||||
return f'<a href="#{_slugify(target[1:])}">{html_mod.escape(display)}</a>'
|
||||
|
||||
# Resolve to another file — link to its exported .html sibling
|
||||
key = Path(target).stem.lower()
|
||||
if key in md_files:
|
||||
href = f"{_slugify(md_files[key].stem)}.html"
|
||||
return f'<a href="{href}">{html_mod.escape(display)}</a>'
|
||||
|
||||
return f'<span class="wikilink-missing">{html_mod.escape(display)}</span>'
|
||||
|
||||
return re.sub(r"\[\[([^\]|]+)(?:\|([^\]]+))?\]\]", _replace, md)
|
||||
|
||||
|
||||
def _render_body(md: str, file_dir: Path, vault_path: Path, current: Path) -> str:
|
||||
"""Render raw markdown to an HTML fragment (images inlined, wikilinks resolved)."""
|
||||
md = _inline_images(md, file_dir, vault_path)
|
||||
md = _convert_wikilinks(md, vault_path, current)
|
||||
# mistune 3.3 types `Markdown.__call__` as `str | list[...]` (le renderer
|
||||
# HTML renvoie toujours `str` à l'exécution).
|
||||
return cast(str, _markdown(md))
|
||||
|
||||
|
||||
def _build_nav(vault_path: Path, current: Path) -> str:
|
||||
"""Build a navigation block linking to every other markdown note in the vault."""
|
||||
items = []
|
||||
for p in _collect_markdown_files(vault_path):
|
||||
if p.resolve() == Path(current).resolve():
|
||||
continue
|
||||
stem = p.stem
|
||||
items.append(f'<li><a href="{_slugify(stem)}.html">{html_mod.escape(stem)}</a></li>')
|
||||
if not items:
|
||||
return ""
|
||||
return (
|
||||
'<nav class="export-nav"><h2>Notes</h2><ul>'
|
||||
+ "".join(items)
|
||||
+ "</ul></nav>"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public exporters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def export_html(vault_path: Path, file_path: Path) -> bytes:
|
||||
"""Convert a markdown note into a standalone HTML document.
|
||||
|
||||
Images are inlined as base64 data URIs, wikilinks become cross-file links,
|
||||
and a navigation block lists every other note in the vault.
|
||||
|
||||
Returns:
|
||||
UTF-8 encoded HTML as bytes.
|
||||
"""
|
||||
vault_path = Path(vault_path)
|
||||
path = _resolve(vault_path, file_path)
|
||||
raw = _read_text(path)
|
||||
body, metadata = _strip_frontmatter(raw)
|
||||
title = str(metadata.get("title") or path.stem)
|
||||
|
||||
rendered = _render_body(body, path.parent, vault_path, path)
|
||||
nav = _build_nav(vault_path, path)
|
||||
|
||||
html_doc = f"""<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="generator" content="ObsiGate">
|
||||
<title>{html_mod.escape(title)}</title>
|
||||
<style>{_HTML_CSS}</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><h1>{html_mod.escape(title)}</h1></header>
|
||||
{nav}
|
||||
{rendered}
|
||||
</body>
|
||||
</html>"""
|
||||
return html_doc.encode("utf-8")
|
||||
|
||||
|
||||
def export_md_bundle(vault_path: Path, directory_path: Path) -> bytes:
|
||||
"""Zip a directory (or single file) of markdown, preserving structure.
|
||||
|
||||
Returns:
|
||||
ZIP archive as bytes.
|
||||
"""
|
||||
vault_path = Path(vault_path)
|
||||
path = _resolve(vault_path, directory_path)
|
||||
|
||||
if not path.exists():
|
||||
raise ExportError(f"Path not found: {path}")
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
if path.is_dir():
|
||||
base = path
|
||||
for p in sorted(path.rglob("*")):
|
||||
if p.is_file():
|
||||
arcname = p.relative_to(base).as_posix()
|
||||
zf.write(p, arcname=arcname)
|
||||
# Avoid emitting an empty (invalid) zip for an empty directory
|
||||
if not any(p.is_file() for p in path.rglob("*")):
|
||||
zf.writestr(".empty", "")
|
||||
else:
|
||||
zf.write(path, arcname=path.name)
|
||||
except OSError as e:
|
||||
raise ExportError(f"Cannot create bundle: {e}") from e
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def export_epub(vault_path: Path, file_path: Path) -> bytes:
|
||||
"""Convert a markdown note into a minimal, valid EPUB 3 container.
|
||||
|
||||
Uses only ``zipfile`` + ``mistune`` (no ``ebooklib`` dependency). Images
|
||||
are inlined as base64 so the ePub is fully self-contained.
|
||||
|
||||
Returns:
|
||||
EPUB archive as bytes.
|
||||
"""
|
||||
vault_path = Path(vault_path)
|
||||
path = _resolve(vault_path, file_path)
|
||||
raw = _read_text(path)
|
||||
body, metadata = _strip_frontmatter(raw)
|
||||
title = str(metadata.get("title") or path.stem)
|
||||
author = str(metadata.get("author") or "ObsiGate")
|
||||
|
||||
rendered = _render_body(body, path.parent, vault_path, path)
|
||||
|
||||
uid = f"obsigate-{_slugify(title)}-{int(datetime.datetime.now(tz=datetime.UTC).timestamp())}"
|
||||
xhtml = f"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="fr" lang="fr">
|
||||
<head>
|
||||
<title>{html_mod.escape(title)}</title>
|
||||
<style>{_HTML_CSS}</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>{html_mod.escape(title)}</h1>
|
||||
{rendered}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
# EPUB requires a valid XHTML-ish body; strip our HTML5-only doctype is not
|
||||
# needed since we emit XML directly above. Escape any bare ampersands in
|
||||
# text nodes outside tags is delegated to mistune's own escaping.
|
||||
content_opf = f"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">{html_mod.escape(uid)}</dc:identifier>
|
||||
<dc:title>{html_mod.escape(title)}</dc:title>
|
||||
<dc:creator>{html_mod.escape(author)}</dc:creator>
|
||||
<dc:language>fr</dc:language>
|
||||
<meta property="dcterms:modified">{datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H:%M:%SZ')}</meta>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
|
||||
<item id="chapter1" href="chapter1.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine toc="ncx">
|
||||
<itemref idref="chapter1"/>
|
||||
</spine>
|
||||
</package>"""
|
||||
|
||||
toc_ncx = f"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
|
||||
<head>
|
||||
<meta name="dtb:uid" content="{html_mod.escape(uid)}"/>
|
||||
</head>
|
||||
<docTitle><text>{html_mod.escape(title)}</text></docTitle>
|
||||
<navMap>
|
||||
<navPoint id="navpoint-1" playOrder="1">
|
||||
<navLabel><text>{html_mod.escape(title)}</text></navLabel>
|
||||
<content src="chapter1.xhtml"/>
|
||||
</navPoint>
|
||||
</navMap>
|
||||
</ncx>"""
|
||||
|
||||
container_xml = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>"""
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
# `mimetype` must be the first entry, stored (uncompressed).
|
||||
zf.writestr(
|
||||
zipfile.ZipInfo("mimetype"),
|
||||
"application/epub+zip",
|
||||
compress_type=zipfile.ZIP_STORED,
|
||||
)
|
||||
zf.writestr("META-INF/container.xml", container_xml)
|
||||
zf.writestr("OEBPS/content.opf", content_opf)
|
||||
zf.writestr("OEBPS/toc.ncx", toc_ncx)
|
||||
zf.writestr("OEBPS/chapter1.xhtml", xhtml)
|
||||
except OSError as e:
|
||||
raise ExportError(f"Cannot create ePub: {e}") from e
|
||||
|
||||
return buffer.getvalue()
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Références croisées ``external_ref`` des notes — Fusion #215 (ACC-1.3).
|
||||
|
||||
Modèle figé par le programme Fusion dans ``dracodev-commons``
|
||||
(``contrats/external_ref.schema.json``, v0.1.0) : la référence vit chez
|
||||
le propriétaire de l'objet source — ici la note ObsiGate, identifiée
|
||||
par ``(vault, path)`` — et pointe vers un objet d'une autre application
|
||||
(projet ou page Flowdeck, tâche ou liste TikTak, carte Kanbante, note
|
||||
ObsiGate) par son URL profonde (``contrats/liens-profonds.md``). Une
|
||||
seule source de vérité par objet : on stocke des références, jamais
|
||||
des copies du contenu cible.
|
||||
|
||||
Stockage : JSON par utilisateur dans ``data/external_refs/``, même
|
||||
patron que :mod:`backend.history` (pas de base de données, cf.
|
||||
``AGENTS.md``). Le répertoire est surchargeable par la variable
|
||||
d'environnement ``OBSIGATE_EXTERNAL_REFS_DIR`` (tests).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, quote, unquote, urlparse
|
||||
|
||||
logger = logging.getLogger("obsigate.external_refs")
|
||||
|
||||
|
||||
def _refs_dir() -> Path:
|
||||
override = os.environ.get("OBSIGATE_EXTERNAL_REFS_DIR")
|
||||
directory = Path(override) if override else Path("data/external_refs")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
return directory
|
||||
|
||||
|
||||
def _refs_file(username: str) -> Path:
|
||||
return _refs_dir() / f"{username}_refs.json"
|
||||
|
||||
|
||||
def _read_refs(username: str) -> list[dict[str, Any]]:
|
||||
file = _refs_file(username)
|
||||
if not file.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(file.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read external refs from {file.name}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _write_refs(username: str, refs: list[dict[str, Any]]) -> None:
|
||||
file = _refs_file(username)
|
||||
try:
|
||||
tmp = file.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(refs, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
shutil.move(str(tmp), str(file))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write external refs to {file.name}: {e}")
|
||||
|
||||
|
||||
# ------------------------------------------------------------ parseur d'URL
|
||||
|
||||
def build_deep_link(target_app: str, target_type: str, target_id: str) -> str | None:
|
||||
"""Construit l'URL profonde d'affichage depuis la **configuration**
|
||||
courante (#223), ou ``None`` si la forme est inconnue.
|
||||
|
||||
Les références stockées portent une URL absolue figée au moment du
|
||||
collage (voir ``add_ref``) : elle peut nommer l'autre environnement
|
||||
(un lien collé en prod, lu en dev). L'affichage ne l'utilise donc
|
||||
jamais telle quelle — il reconstruit l'URL depuis
|
||||
``(target_app, target_type, target_id)`` et les bases publiques
|
||||
configurées (``backend.draco_config``), au moment de la lecture.
|
||||
Aucune donnée stockée n'est migrée ni réécrite.
|
||||
"""
|
||||
from backend.draco_config import app_url
|
||||
|
||||
if target_app == "tiktak":
|
||||
if target_type == "task":
|
||||
return f"{app_url('tiktak')}/?task={quote(target_id, safe='')}"
|
||||
if target_type == "list":
|
||||
return f"{app_url('tiktak')}/?list={quote(target_id, safe='')}"
|
||||
return None
|
||||
if target_app == "obsigate" and target_type == "note":
|
||||
return f"{app_url('obsigate')}/#file={quote(target_id, safe=':')}"
|
||||
if target_app == "flowdeck":
|
||||
if target_type == "page":
|
||||
return f"{app_url('flowdeck')}/pages/{quote(target_id, safe='')}"
|
||||
if target_type == "project":
|
||||
return f"{app_url('flowdeck')}/collections/{quote(target_id, safe='')}"
|
||||
return None
|
||||
if target_app == "kanbante" and target_type == "card":
|
||||
m = re.fullmatch(r"([^/]+)/([^/#]+)#(\d+)", target_id or "")
|
||||
if m:
|
||||
owner, repo, num = m.groups()
|
||||
return f"{app_url('gitea')}/{owner}/{repo}/issues/{num}"
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _avec_url_affichee(ref: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Copie de la référence dont ``target_url`` est reconstruite
|
||||
depuis la config courante (#223) — le stockage n'est pas touché."""
|
||||
construite = build_deep_link(
|
||||
str(ref.get("target_app") or ""),
|
||||
str(ref.get("target_type") or ""),
|
||||
str(ref.get("target_id") or ""),
|
||||
)
|
||||
if construite is None:
|
||||
return ref
|
||||
copie = dict(ref)
|
||||
copie["target_url"] = construite
|
||||
return copie
|
||||
|
||||
|
||||
def parse_deep_link(url: str) -> tuple[str, str, str, str] | None:
|
||||
"""Reconnaît une URL profonde des applications dracodev.
|
||||
|
||||
Formes figées par ``dracodev-commons/contrats/liens-profonds.md``.
|
||||
Retourne ``(target_app, target_type, target_id, label_par_défaut)``
|
||||
ou ``None`` si l'URL ne correspond à aucune forme du contrat.
|
||||
|
||||
Hôtes reconnus (#223) : les hôtes **officiels** et les hôtes
|
||||
**configurés** de chaque application (``backend.draco_config``) —
|
||||
un lien collé depuis l'environnement courant (noms en ``-dev``
|
||||
déclarés dans son ``.env``) comme depuis l'autre environnement
|
||||
est donc reconnu ; aucun nom d'environnement n'est figé ici.
|
||||
"""
|
||||
from backend.draco_config import hotes_reconnus
|
||||
|
||||
try:
|
||||
u = urlparse((url or "").strip())
|
||||
except ValueError:
|
||||
return None
|
||||
host = (u.hostname or "").lower()
|
||||
app = hotes_reconnus().get(host)
|
||||
if app == "tiktak":
|
||||
q = parse_qs(u.query)
|
||||
if q.get("task"):
|
||||
return ("tiktak", "task", q["task"][0], f"TikTak · tâche {q['task'][0]}")
|
||||
if q.get("list"):
|
||||
return ("tiktak", "list", q["list"][0], f"TikTak · liste {q['list'][0]}")
|
||||
return None
|
||||
if app == "obsigate":
|
||||
frag = u.fragment or ""
|
||||
if frag.startswith("file="):
|
||||
cible = unquote(frag[5:])
|
||||
if ":" in cible:
|
||||
nom = cible.rsplit("/", 1)[-1]
|
||||
return ("obsigate", "note", cible, f"ObsiGate · {nom}")
|
||||
return None
|
||||
if app == "flowdeck":
|
||||
m = re.fullmatch(r"/pages/(\d+)", u.path or "")
|
||||
if m:
|
||||
return ("flowdeck", "page", m.group(1), f"Flowdeck · page {m.group(1)}")
|
||||
m = re.fullmatch(r"/collections/(\d+)", u.path or "")
|
||||
if m:
|
||||
return ("flowdeck", "project", m.group(1), f"Flowdeck · projet {m.group(1)}")
|
||||
return None
|
||||
if app == "gitea":
|
||||
m = re.fullmatch(r"/([^/]+)/([^/]+)/issues/(\d+)", u.path or "")
|
||||
if m:
|
||||
owner, repo, num = m.groups()
|
||||
return ("kanbante", "card", f"{owner}/{repo}#{num}", f"Kanbante · {repo} #{num}")
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
# ------------------------------------------------------------ CRUD
|
||||
|
||||
def list_refs(username: str, vault: str, path: str) -> list[dict[str, Any]]:
|
||||
"""Références posées sur une note, dans l'ordre de création.
|
||||
|
||||
``target_url`` de chaque référence retournée est **reconstruite
|
||||
depuis la config courante** (#223, ``_avec_url_affichee``) : la
|
||||
valeur stockée, figée au collage, n'est jamais servie telle quelle.
|
||||
"""
|
||||
if not username:
|
||||
return []
|
||||
return [
|
||||
_avec_url_affichee(r) for r in _read_refs(username)
|
||||
if r.get("owner_vault") == vault and r.get("owner_path") == path
|
||||
]
|
||||
|
||||
|
||||
def get_ref(username: str, ref_id: str) -> dict[str, Any] | None:
|
||||
"""Une référence par son identifiant, ou ``None`` (URL reconstruite, #223)."""
|
||||
for r in _read_refs(username):
|
||||
if r.get("id") == ref_id:
|
||||
return _avec_url_affichee(r)
|
||||
return None
|
||||
|
||||
|
||||
def add_ref(
|
||||
username: str,
|
||||
vault: str,
|
||||
path: str,
|
||||
target_app: str,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
target_url: str,
|
||||
label: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Ajoute une référence à une note et la retourne (avec son ``id``)."""
|
||||
refs = _read_refs(username)
|
||||
ref = {
|
||||
"id": uuid.uuid4().hex,
|
||||
"owner_type": "obsigate.note",
|
||||
"owner_id": f"{vault}:{path}",
|
||||
"owner_vault": vault,
|
||||
"owner_path": path,
|
||||
"target_app": target_app,
|
||||
"target_type": target_type,
|
||||
"target_id": target_id,
|
||||
"target_url": target_url,
|
||||
"label": label,
|
||||
"domain": None,
|
||||
"status": "ok",
|
||||
"created_by": "user",
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
refs.append(ref)
|
||||
_write_refs(username, refs)
|
||||
return ref
|
||||
|
||||
|
||||
def delete_ref(username: str, ref_id: str) -> dict[str, Any] | None:
|
||||
"""Retire une référence ; retourne la référence retirée ou ``None``."""
|
||||
refs = _read_refs(username)
|
||||
kept = [r for r in refs if r.get("id") != ref_id]
|
||||
if len(kept) == len(refs):
|
||||
return None
|
||||
removed = next(r for r in refs if r.get("id") == ref_id)
|
||||
_write_refs(username, kept)
|
||||
return _avec_url_affichee(removed)
|
||||
@@ -0,0 +1,430 @@
|
||||
# backend/file_chat.py — historique de discussion (#169, #190)
|
||||
"""Chat history persisted under ``data/chats/``.
|
||||
|
||||
One JSON document per ``(vault, path)`` pair, keyed by a SHA-256 of both so
|
||||
the filename never carries user-controlled path separators. Writes are
|
||||
atomic (tmp + move) and the message list is capped at
|
||||
:data:`MAX_MESSAGES` to bound growth.
|
||||
|
||||
#190 adds the **general chat**: the same store addressed with the reserved
|
||||
sentinels (:data:`GLOBAL_VAULT` / :data:`GLOBAL_PATH`), so no second
|
||||
implementation. Messages may carry an ``attachment`` (image/video/url)
|
||||
uploaded under ``data/chat_uploads/``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.render import _render_markdown
|
||||
|
||||
logger = logging.getLogger("obsigate.file_chat")
|
||||
|
||||
CHAT_DIR = Path("data/chats")
|
||||
MAX_MESSAGES = 500 # retention ceiling per file (oldest dropped first)
|
||||
MAX_TEXT = 4000 # characters per message
|
||||
|
||||
# ponytail: global lock over read-modify-write — chat writes are HTTP-only and
|
||||
# serialized anyway, this just makes losing a message to a future thread (or a
|
||||
# watcher hook) impossible; per-file locks if it ever becomes contended.
|
||||
_LOCK = threading.RLock()
|
||||
|
||||
# #190 — general (non file-bound) conversation, stored like any other one.
|
||||
GLOBAL_VAULT = "__global__"
|
||||
GLOBAL_PATH = "general"
|
||||
|
||||
# #191 — private (2 users) conversations reuse the same store: the vault is
|
||||
# the reserved sentinel and the path is the sorted username pair, so the
|
||||
# storage key never depends on who asks.
|
||||
DM_VAULT = "__dm__"
|
||||
|
||||
# #190 — attachments (image/video) live outside the vaults.
|
||||
UPLOAD_DIR = Path("data/chat_uploads")
|
||||
MAX_UPLOAD_BYTES = 25 * 1024 * 1024 # 25 MB per attachment
|
||||
ALLOWED_ATTACH_EXT = {
|
||||
".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg",
|
||||
".mp4", ".webm", ".ogg", ".mov", ".m4v",
|
||||
}
|
||||
|
||||
# #191 — link preview fetch budget
|
||||
PREVIEW_TIMEOUT = 5.0 # seconds
|
||||
PREVIEW_MAX_BYTES = 512 * 1024 # only the head of the page is parsed
|
||||
PREVIEW_IMAGE_MAX = 2 * 1024 * 1024 # ponytail: 2 MB ceiling on a thumbnail
|
||||
|
||||
|
||||
def _chat_file(vault: str, path: str) -> Path:
|
||||
"""Return the chat file for *(vault, path)* (hashed, traversal-proof)."""
|
||||
CHAT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
key = hashlib.sha256(f"{vault}\0{path}".encode()).hexdigest()[:32]
|
||||
return CHAT_DIR / f"{key}.json"
|
||||
|
||||
|
||||
def _read(vault: str, path: str) -> dict[str, Any]:
|
||||
"""Load the raw chat document (empty structure when missing/corrupt)."""
|
||||
file = _chat_file(vault, path)
|
||||
if not file.exists():
|
||||
return {"vault": vault, "path": path, "messages": []}
|
||||
try:
|
||||
doc = json.loads(file.read_text(encoding="utf-8"))
|
||||
if not isinstance(doc.get("messages"), list):
|
||||
raise TypeError("messages is not a list") # caught by the handler below
|
||||
return doc
|
||||
except Exception as e:
|
||||
logger.error("Failed to read chat for %s/%s: %s", vault, path, e)
|
||||
return {"vault": vault, "path": path, "messages": []}
|
||||
|
||||
|
||||
def _write(file: Path, doc: dict[str, Any]) -> None:
|
||||
"""Atomically persist *doc* (tmp file + rename)."""
|
||||
try:
|
||||
tmp = file.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
shutil.move(str(tmp), str(file))
|
||||
except Exception as e:
|
||||
logger.error("Failed to write chat %s: %s", file.name, e)
|
||||
|
||||
|
||||
def get_messages(vault: str, path: str) -> list[dict[str, Any]]:
|
||||
"""Return the chat history for *(vault, path)* (chronological).
|
||||
|
||||
Every message carries its rendered ``html`` (#193): same markdown pipeline
|
||||
as a document (mistune + sanitizer), computed on read so a template change
|
||||
applies to the whole history without rewriting the JSON store.
|
||||
"""
|
||||
return [_decorate(m, vault) for m in _read(vault, path).get("messages", [])]
|
||||
|
||||
|
||||
def _decorate(msg: dict[str, Any], vault: str) -> dict[str, Any]:
|
||||
"""Return a copy of *msg* with its sanitized markdown ``html`` (#193).
|
||||
|
||||
The stored message is left untouched (``html`` is never persisted). A
|
||||
rendering failure must never break the chat: the message goes out with an
|
||||
empty ``html`` and the client falls back to plain text.
|
||||
"""
|
||||
try:
|
||||
html = _render_markdown(msg.get("text", ""), vault)
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
logger.warning("chat markdown rendering failed: %s", e)
|
||||
html = ""
|
||||
return {**msg, "html": html}
|
||||
|
||||
|
||||
def add_message(
|
||||
vault: str,
|
||||
path: str,
|
||||
user: str,
|
||||
text: str,
|
||||
attachment: dict[str, Any] | None = None,
|
||||
preview: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a message and persist it. Returns the stored message.
|
||||
|
||||
The list is capped at :data:`MAX_MESSAGES` (oldest dropped first).
|
||||
*attachment* (#190) is ``{name, url, mime, kind}`` for image/video/url;
|
||||
*preview* (#191) is the OpenGraph card of the first URL in *text*.
|
||||
"""
|
||||
text = (text or "").strip()[:MAX_TEXT]
|
||||
msg: dict[str, Any] = {
|
||||
"id": uuid.uuid4().hex[:12],
|
||||
"user": user or "anonyme",
|
||||
"text": text,
|
||||
"ts": time.time(),
|
||||
}
|
||||
if attachment:
|
||||
msg["attachment"] = attachment
|
||||
if preview:
|
||||
msg["preview"] = preview
|
||||
return _append(vault, path, msg)
|
||||
|
||||
|
||||
def _append(
|
||||
vault: str,
|
||||
path: str,
|
||||
msg: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Cap, persist and return *msg* (shared by file, global and DM chats).
|
||||
|
||||
The read-modify-write of the whole document happens under ``_LOCK`` so a
|
||||
concurrent writer can never drop a message (same class of bug as
|
||||
BUG-029 on ``users.json``).
|
||||
"""
|
||||
with _LOCK:
|
||||
doc = _read(vault, path)
|
||||
messages = list(doc.get("messages", []))
|
||||
messages.append(msg)
|
||||
if len(messages) > MAX_MESSAGES:
|
||||
messages = messages[-MAX_MESSAGES:]
|
||||
doc["messages"] = messages
|
||||
_write(_chat_file(vault, path), doc)
|
||||
return _decorate(msg, vault) # #193 — le html part avec l'écho SSE
|
||||
|
||||
|
||||
# --- #192 : accusé de réception ---------------------------------------------
|
||||
|
||||
def get_read(vault: str, path: str) -> dict[str, float]:
|
||||
"""Return ``{username: last_read_ts}`` for a conversation (#192)."""
|
||||
return {str(u): float(ts) for u, ts in (_read(vault, path).get("read") or {}).items()}
|
||||
|
||||
|
||||
def mark_read(vault: str, path: str, user: str) -> dict[str, float]:
|
||||
"""Record that *user* has seen the conversation (#192).
|
||||
|
||||
Returns the whole read map so the caller can broadcast it on SSE: a
|
||||
sender learns their messages were received as soon as the recipient
|
||||
displays the conversation.
|
||||
"""
|
||||
if not user:
|
||||
return get_read(vault, path)
|
||||
with _LOCK:
|
||||
doc = _read(vault, path)
|
||||
read = {str(u): float(ts) for u, ts in (doc.get("read") or {}).items()}
|
||||
read[user] = time.time()
|
||||
doc["read"] = read
|
||||
_write(_chat_file(vault, path), doc)
|
||||
return read
|
||||
|
||||
|
||||
# --- #191 : messages privés (2 utilisateurs) -------------------------------
|
||||
|
||||
def dm_path(user_a: str, user_b: str) -> str:
|
||||
"""Storage path for the private conversation between two users.
|
||||
|
||||
The pair is sorted so both participants address the same document.
|
||||
"""
|
||||
return "|".join(sorted([user_a, user_b]))
|
||||
|
||||
|
||||
def get_dm_messages(user_a: str, user_b: str) -> list[dict[str, Any]]:
|
||||
"""Return the private history between two users (chronological)."""
|
||||
return get_messages(DM_VAULT, dm_path(user_a, user_b))
|
||||
|
||||
|
||||
def add_dm_message(
|
||||
user_a: str,
|
||||
user_b: str,
|
||||
author: str,
|
||||
text: str,
|
||||
attachment: dict[str, Any] | None = None,
|
||||
preview: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a private message. Returns the stored message."""
|
||||
return add_message(DM_VAULT, dm_path(user_a, user_b), author, text, attachment, preview)
|
||||
|
||||
|
||||
# --- #191 : suppression -----------------------------------------------------
|
||||
|
||||
def delete_message(vault: str, path: str, message_id: str) -> bool:
|
||||
"""Remove one message from a conversation. True when it existed."""
|
||||
with _LOCK:
|
||||
doc = _read(vault, path)
|
||||
messages = list(doc.get("messages", []))
|
||||
kept = [m for m in messages if m.get("id") != message_id]
|
||||
if len(kept) == len(messages):
|
||||
return False
|
||||
doc["messages"] = kept
|
||||
_write(_chat_file(vault, path), doc)
|
||||
return True
|
||||
|
||||
|
||||
# --- #190 : chat général (conversation centrale, hors fichier) -------------
|
||||
|
||||
def get_global_messages() -> list[dict[str, Any]]:
|
||||
"""Return the general-chat history (chronological)."""
|
||||
return get_messages(GLOBAL_VAULT, GLOBAL_PATH)
|
||||
|
||||
|
||||
def add_global_message(
|
||||
user: str,
|
||||
text: str,
|
||||
attachment: dict[str, Any] | None = None,
|
||||
preview: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a message to the general chat. Returns the stored message."""
|
||||
return add_message(GLOBAL_VAULT, GLOBAL_PATH, user, text, attachment, preview)
|
||||
|
||||
|
||||
def save_attachment(filename: str, data: bytes) -> dict[str, Any]:
|
||||
"""Persist an uploaded attachment under :data:`UPLOAD_DIR`.
|
||||
|
||||
Returns ``{name, url, mime, kind}``. The stored name is a fresh UUID
|
||||
(never the client name), the extension must be in
|
||||
:data:`ALLOWED_ATTACH_EXT` and the size is capped at
|
||||
:data:`MAX_UPLOAD_BYTES`.
|
||||
|
||||
Raises:
|
||||
ValueError: extension refused, empty file or size exceeded.
|
||||
"""
|
||||
ext = Path(filename or "").suffix.lower()
|
||||
if ext not in ALLOWED_ATTACH_EXT:
|
||||
raise ValueError(f"extension refusée : {ext or '(aucune)'}")
|
||||
if not data:
|
||||
raise ValueError("fichier vide")
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise ValueError(f"fichier trop lourd (max {MAX_UPLOAD_BYTES // (1024 * 1024)} MB)")
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
name = f"{uuid.uuid4().hex}{ext}"
|
||||
(UPLOAD_DIR / name).write_bytes(data)
|
||||
kind = "video" if ext in {".mp4", ".webm", ".ogg", ".mov", ".m4v"} else "image"
|
||||
return {
|
||||
"name": name,
|
||||
"url": f"/api/chat/attachment/{name}",
|
||||
"mime": _MIME_BY_EXT.get(ext, "application/octet-stream"),
|
||||
"kind": kind,
|
||||
}
|
||||
|
||||
|
||||
# --- #191 : link preview ----------------------------------------------------
|
||||
|
||||
_URL_RE = re.compile(r"https?://[^\s<>\"']+")
|
||||
_PREVIEW_CACHE: dict[str, dict[str, Any] | None] = {}
|
||||
PREVIEW_CACHE_MAX = 200
|
||||
|
||||
|
||||
def _og(content: str, prop: str) -> str:
|
||||
"""Extract one OpenGraph/``<title>`` value from an HTML head (regex)."""
|
||||
for pattern in (
|
||||
rf'<meta[^>]+(?:property|name)="{prop}"[^>]+content="([^"]*)"',
|
||||
rf'<meta[^>]+content="([^"]*)"[^>]+(?:property|name)="{prop}"',
|
||||
):
|
||||
m = re.search(pattern, content, re.IGNORECASE)
|
||||
if m:
|
||||
return html.unescape(m.group(1)).strip()[:300]
|
||||
if prop == "og:title":
|
||||
m = re.search(r"<title[^>]*>([^<]*)</title>", content, re.IGNORECASE)
|
||||
if m:
|
||||
return html.unescape(m.group(1)).strip()[:300]
|
||||
return ""
|
||||
|
||||
|
||||
# BUG-109 — content-type → extension (the attachment allow-list decides).
|
||||
_IMG_EXT_BY_MIME = {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
"image/svg+xml": ".svg",
|
||||
}
|
||||
|
||||
|
||||
def _proxy_image(img_url: str, page_url: str) -> str:
|
||||
"""Download *img_url* into ``chat_uploads`` and return a same-origin URL.
|
||||
|
||||
The response CSP is ``img-src 'self' data: blob:``: a remote ``og:image``
|
||||
would be blocked by the browser (BUG-109). Relative and protocol-relative
|
||||
values are resolved against *page_url* first. Raises ``ValueError`` /
|
||||
``SSRFError`` on any failure — the caller keeps the card and drops only
|
||||
the thumbnail.
|
||||
"""
|
||||
from backend.tools.web import USER_AGENT, _assert_public_http_url
|
||||
|
||||
full = urljoin(page_url, img_url)
|
||||
_assert_public_http_url(full)
|
||||
resp = httpx.get(
|
||||
full,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=PREVIEW_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ValueError(f"HTTP {resp.status_code}")
|
||||
data = resp.content
|
||||
if not data or len(data) > PREVIEW_IMAGE_MAX:
|
||||
raise ValueError("image vide ou trop lourde")
|
||||
mime = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||||
ext = _IMG_EXT_BY_MIME.get(mime) or Path(urlparse(full).path).suffix.lower()
|
||||
# save_attachment(): allow-list d'extensions + nom UUID (jamais le nom distant)
|
||||
return str(save_attachment(f"preview{ext}", data)["url"])
|
||||
|
||||
|
||||
def build_preview(text: str) -> dict[str, Any] | None:
|
||||
"""Fetch OpenGraph metadata for the first URL in *text* (#191).
|
||||
|
||||
SSRF-guarded (reuses the web-tool guard), size/time capped, cached in a
|
||||
bounded dict. Returns ``{url, title, description, image, site}`` or
|
||||
``None`` when there is no URL / the fetch fails (never raises: a dead
|
||||
link must not block the message). ``image`` is a **same-origin**
|
||||
``/api/chat/attachment/...`` URL (BUG-109), empty when the thumbnail
|
||||
could not be fetched.
|
||||
"""
|
||||
m = _URL_RE.search(text or "")
|
||||
if not m:
|
||||
return None
|
||||
url = m.group(0).rstrip(".,;:!?)")
|
||||
if url in _PREVIEW_CACHE:
|
||||
cached = _PREVIEW_CACHE[url]
|
||||
return dict(cached) if cached else None
|
||||
try:
|
||||
from backend.tools.web import USER_AGENT, _assert_public_http_url
|
||||
|
||||
_assert_public_http_url(url)
|
||||
resp = httpx.get(
|
||||
url,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "text/html,*/*"},
|
||||
timeout=PREVIEW_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ValueError(f"HTTP {resp.status_code}")
|
||||
body = resp.text[:PREVIEW_MAX_BYTES]
|
||||
# BUG-109 : vignette téléchargée côté serveur — une image distante
|
||||
# échouerait à la CSP. Échec isolé = carte sans vignette.
|
||||
image = _og(body, "og:image")
|
||||
try:
|
||||
image = _proxy_image(image, url) if image else ""
|
||||
except Exception as ie:
|
||||
logger.debug("preview image failed for %s: %s", url, ie)
|
||||
image = ""
|
||||
preview = {
|
||||
"url": url,
|
||||
"title": _og(body, "og:title") or _og(body, "og:site_name"),
|
||||
"description": _og(body, "og:description"),
|
||||
"image": image,
|
||||
"site": _og(body, "og:site_name") or (url.split("/")[2] if "/" in url[8:] else url),
|
||||
}
|
||||
if not preview["title"]:
|
||||
raise ValueError("pas de titre")
|
||||
except Exception as e:
|
||||
logger.debug("link preview failed for %s: %s", url, e)
|
||||
preview = None
|
||||
if len(_PREVIEW_CACHE) >= PREVIEW_CACHE_MAX:
|
||||
_PREVIEW_CACHE.pop(next(iter(_PREVIEW_CACHE))) # oldest first (dict order)
|
||||
_PREVIEW_CACHE[url] = preview
|
||||
return dict(preview) if preview else None
|
||||
|
||||
|
||||
def attachment_path(name: str) -> Path | None:
|
||||
"""Resolve an attachment by its stored name (UUID+ext only, no traversal)."""
|
||||
p = Path(name)
|
||||
if p.name != name or p.suffix.lower() not in ALLOWED_ATTACH_EXT:
|
||||
return None
|
||||
file = UPLOAD_DIR / p.name
|
||||
return file if file.exists() else None
|
||||
|
||||
|
||||
# Extension → MIME (literals only; ``mimetypes`` guesses poorly for a few).
|
||||
_MIME_BY_EXT = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".svg": "image/svg+xml",
|
||||
".mp4": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".ogg": "video/ogg",
|
||||
".mov": "video/quicktime",
|
||||
".m4v": "video/x-m4v",
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
"""Génération du Guide d'utilisation téléchargeable en Markdown et PDF (#105).
|
||||
|
||||
Source unique de vérité : la modale ``#help-modal`` de ``frontend/index.html``
|
||||
(comme dans l'application) + les blocs ``data-i18n`` résolus dans les locales
|
||||
``frontend/locales/{fr,en}.json`` — le téléchargement reflète donc exactement
|
||||
ce que voit l'utilisateur, dans sa langue.
|
||||
|
||||
Le Markdown est produit par un convertisseur HTML→MD minimal (stdlib) ; le
|
||||
PDF passe par le moteur d'export existant (WeasyPrint) avec repli reportlab
|
||||
quand les bibliothèques natives GTK manquent (Windows).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("obsigate.guide")
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
INDEX_HTML = ROOT / "frontend" / "index.html"
|
||||
LOCALES_DIR = ROOT / "frontend" / "locales"
|
||||
VERSION_FILE = ROOT / "VERSION"
|
||||
DIAGRAMS_DIR = ROOT / "backend" / "assets" / "guide_diagrams"
|
||||
|
||||
|
||||
def diagram_png_for(code: str) -> Path | None:
|
||||
"""Chemin du PNG pré-rendu (scripts/build_guide_diagrams.py) pour un code
|
||||
Mermaid, ou None. Le hash doit rester synchrone avec le script de build :
|
||||
sha1(unescape(code).strip())[:16]."""
|
||||
normalized = html.unescape(code).strip()
|
||||
# Identifiant de cache déterministe (pas un usage sécurité).
|
||||
sha = hashlib.sha1(normalized.encode("utf-8")).hexdigest()[:16] # nosec B324
|
||||
png = DIAGRAMS_DIR / (sha + ".png")
|
||||
return png if png.exists() else None
|
||||
|
||||
# Éléments décoratifs exclus des exports
|
||||
_SKIP_CLASSES = {"help-hero-visual", "editor-modal", "help-nav"}
|
||||
# En-tête HTML du guide (mode lecture)
|
||||
_HEADER_BLOCK = "ObsiGate User Guide"
|
||||
|
||||
|
||||
class Node:
|
||||
"""Noeud DOM minimal (stdlib only)."""
|
||||
|
||||
__slots__ = ("attrs", "children", "parent", "tag")
|
||||
|
||||
def __init__(self, tag: str, attrs: dict[str, str | None], parent: Node | None = None):
|
||||
self.tag = tag
|
||||
self.attrs = attrs
|
||||
self.children: list[Node | str] = []
|
||||
self.parent = parent
|
||||
|
||||
def cls(self) -> str:
|
||||
return self.attrs.get("class") or ""
|
||||
|
||||
def i18n(self) -> str | None:
|
||||
v = self.attrs.get("data-i18n")
|
||||
return v if isinstance(v, str) else None
|
||||
|
||||
def find_all(self, tag: str) -> list[Node]:
|
||||
out: list[Node] = []
|
||||
for c in self.children:
|
||||
if isinstance(c, Node):
|
||||
if c.tag == tag:
|
||||
out.append(c)
|
||||
out.extend(c.find_all(tag))
|
||||
return out
|
||||
|
||||
|
||||
_VOID_TAGS = {"br", "img", "hr", "input", "meta", "link"}
|
||||
|
||||
|
||||
class _TreeBuilder(HTMLParser):
|
||||
"""Constructeur d'arbre tolérant (ignore les balises orphelines)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = Node("#root", {})
|
||||
self.cur = self.root
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
a = {k: v for k, v in attrs}
|
||||
node = Node(tag, a, self.cur)
|
||||
self.cur.children.append(node)
|
||||
if tag not in _VOID_TAGS:
|
||||
self.cur = node
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
a = {k: v for k, v in attrs}
|
||||
self.cur.children.append(Node(tag, a, self.cur))
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
n: Node | None = self.cur
|
||||
while n is not None and n.tag != tag:
|
||||
n = n.parent
|
||||
if n is not None and n.parent is not None:
|
||||
self.cur = n.parent
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self.cur.children.append(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction / cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_cache: dict[tuple[str, str], tuple[tuple[float, int, float, int], bytes]] = {}
|
||||
|
||||
|
||||
def _read_index_html() -> str:
|
||||
return INDEX_HTML.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _guide_fragment(index_html: str) -> str:
|
||||
"""Le HTML de #help-modal…help-content jusqu'au footer du guide."""
|
||||
start = index_html.index('id="help-modal"')
|
||||
cstart = index_html.index('<div class="help-content">', start)
|
||||
end = index_html.index('<div class="help-footer">', cstart)
|
||||
return index_html[cstart:end]
|
||||
|
||||
|
||||
def _locale_strings(lang: str) -> dict[str, str]:
|
||||
path = LOCALES_DIR / (lang if lang in ("fr", "en") else "fr")
|
||||
return json.loads(Path(path).with_suffix(".json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _signature() -> tuple[float, int, float, int]:
|
||||
st = INDEX_HTML.stat()
|
||||
lt = (LOCALES_DIR / "fr.json").stat()
|
||||
return (st.st_mtime, st.st_size, lt.st_mtime, lt.st_size)
|
||||
|
||||
|
||||
def _app_version() -> str:
|
||||
try:
|
||||
return VERSION_FILE.read_text(encoding="utf-8").strip() or "dev"
|
||||
except OSError:
|
||||
return "dev"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Résolution i18n : un node portant data-i18n est REMPLACÉ par le contenu
|
||||
# (HTML) de la locale — exactement comme _applyDOM() dans le navigateur.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_i18n(node: Node, loc: dict[str, str]) -> list[Node | str]:
|
||||
"""Retourne les children effectifs d'un node (locale si data-i18n[-html])."""
|
||||
key = node.i18n() or node.attrs.get("data-i18n-html")
|
||||
if not isinstance(key, str):
|
||||
return node.children
|
||||
value = loc.get(key)
|
||||
if value is None:
|
||||
# clé absente de la locale : garder le texte FR inline de index.html
|
||||
return node.children
|
||||
tb = _TreeBuilder()
|
||||
tb.feed(f"<span>{value}</span>")
|
||||
span = tb.root.children[0]
|
||||
assert isinstance(span, Node)
|
||||
return span.children
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_WS_RE = re.compile(r"[ \t]*\n[ \t]*")
|
||||
|
||||
|
||||
def _collapse(text: str) -> str:
|
||||
return _WS_RE.sub(" ", text).strip()
|
||||
|
||||
|
||||
def _md_inline(node: Node | str, loc: dict[str, str]) -> str:
|
||||
if isinstance(node, str):
|
||||
return _collapse(node)
|
||||
tag = node.tag
|
||||
kids = _resolve_i18n(node, loc)
|
||||
inner = "".join(_md_inline(c, loc) for c in kids)
|
||||
if tag == "br":
|
||||
return " "
|
||||
if tag in ("strong", "b"):
|
||||
t = inner.strip()
|
||||
return f"**{t}**" if t else ""
|
||||
if tag in ("em", "i"):
|
||||
if node.cls().startswith("lucide") or tag == "i" and not inner.strip():
|
||||
return ""
|
||||
t = inner.strip()
|
||||
return f"*{t}*" if t else ""
|
||||
if tag == "code":
|
||||
t = inner.replace("`", "'").strip()
|
||||
return f"`{t}`" if t else ""
|
||||
if tag == "kbd":
|
||||
t = inner.strip()
|
||||
return f"`{t}`" if t else ""
|
||||
if tag == "a":
|
||||
href = node.attrs.get("href") or ""
|
||||
t = inner.strip()
|
||||
if href.startswith("http") and t:
|
||||
return f"[{t}]({href})"
|
||||
return t
|
||||
if tag == "img":
|
||||
alt = node.attrs.get("alt") or ""
|
||||
return f"![{alt}]"
|
||||
return inner
|
||||
|
||||
|
||||
def _md_block(node: Node | str, out: list[str], loc: dict[str, str], depth: int = 0) -> None:
|
||||
"""Remplit ``out`` (bloc courant) — ``pending`` gère listes imbriquées."""
|
||||
if isinstance(node, str):
|
||||
t = _collapse(node)
|
||||
if t:
|
||||
out.append(t)
|
||||
return
|
||||
if any(c in node.cls().split() for c in _SKIP_CLASSES):
|
||||
return
|
||||
tag = node.tag
|
||||
|
||||
if tag == "pre":
|
||||
raw = _pre_text(node)
|
||||
lang = "mermaid" if "mermaid" in raw[:40] or "language-mermaid" in _pre_classes(node) else ""
|
||||
out.append(f"```{lang}\n{raw.rstrip()}\n```")
|
||||
return
|
||||
|
||||
kids = _resolve_i18n(node, loc)
|
||||
|
||||
if tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
level = int(tag[1])
|
||||
text = _collapse("".join(_md_inline(c, loc) for c in kids))
|
||||
if text:
|
||||
out.append("#" * level + " " + text)
|
||||
return
|
||||
|
||||
if tag == "p":
|
||||
text = _collapse("".join(_md_inline(c, loc) for c in kids))
|
||||
if text:
|
||||
out.append(text)
|
||||
return
|
||||
|
||||
if tag in ("ul", "ol"):
|
||||
_md_list(kids, out, loc, tag, depth)
|
||||
return
|
||||
|
||||
if tag == "table":
|
||||
_md_table(node, out, loc)
|
||||
return
|
||||
|
||||
# conteneurs neutres (section, div, span de bloc, li imbriqué…)
|
||||
for c in kids:
|
||||
_md_block(c, out, loc, depth)
|
||||
|
||||
|
||||
def _md_list(items: list[Node | str], out: list[str], loc: dict[str, str], kind: str, depth: int) -> None:
|
||||
n = 0
|
||||
for li in items:
|
||||
if isinstance(li, str):
|
||||
continue
|
||||
if li.tag == "li":
|
||||
n += 1
|
||||
marker = "- " if kind == "ul" else f"{n}. "
|
||||
text_parts: list[str] = []
|
||||
nested: list[Node] = []
|
||||
for c in li.children:
|
||||
if isinstance(c, Node) and c.tag in ("ul", "ol"):
|
||||
nested.append(c)
|
||||
else:
|
||||
text_parts.append(_md_inline(c, loc))
|
||||
line = _collapse("".join(text_parts))
|
||||
if line:
|
||||
out.append(" " * depth + marker + line)
|
||||
for sub in nested:
|
||||
_md_list(sub.children, out, loc, sub.tag, depth + 1)
|
||||
elif li.tag in ("ul", "ol"):
|
||||
_md_list(li.children, out, loc, li.tag, depth)
|
||||
|
||||
|
||||
def _md_table(node: Node, out: list[str], loc: dict[str, str]) -> None:
|
||||
rows = node.find_all("tr")
|
||||
if not rows:
|
||||
return
|
||||
grid: list[list[str]] = []
|
||||
for tr in rows:
|
||||
cells = []
|
||||
for td in tr.children:
|
||||
if isinstance(td, Node) and td.tag in ("td", "th"):
|
||||
cells.append(_collapse("".join(_md_inline(c, loc) for c in td.children)).replace("|", "\\|") or " ")
|
||||
if cells:
|
||||
grid.append(cells)
|
||||
if not grid:
|
||||
return
|
||||
width = max(len(r) for r in grid)
|
||||
grid = [r + [" "] * (width - len(r)) for r in grid]
|
||||
out.append("| " + " | ".join(grid[0]) + " |")
|
||||
out.append("|" + "|".join([" --- "] * width) + "|")
|
||||
for r in grid[1:]:
|
||||
out.append("| " + " | ".join(r) + " |")
|
||||
|
||||
|
||||
def _pre_text(node: Node) -> str:
|
||||
"""Texte brut préservé d'un <pre> (les locales n'y touchent pas)."""
|
||||
buf: list[str] = []
|
||||
|
||||
def walk(n: Node | str) -> None:
|
||||
if isinstance(n, str):
|
||||
buf.append(n)
|
||||
return
|
||||
for c in n.children:
|
||||
walk(c)
|
||||
|
||||
walk(node)
|
||||
return "".join(buf).strip("\n")
|
||||
|
||||
|
||||
def _pre_classes(node: Node) -> str:
|
||||
cls = node.cls()
|
||||
for c in node.find_all("code"):
|
||||
cls += " " + c.cls()
|
||||
return cls
|
||||
|
||||
|
||||
def build_guide_markdown(lang: str = "fr") -> bytes:
|
||||
"""Guide complet en Markdown (UTF-8), dans la langue demandée."""
|
||||
index_html = _read_index_html()
|
||||
loc = _locale_strings(lang)
|
||||
tree = _TreeBuilder()
|
||||
tree.feed(_guide_fragment(index_html))
|
||||
root = tree.root.children[0]
|
||||
assert isinstance(root, Node)
|
||||
|
||||
blocks: list[str] = []
|
||||
content = _guide_title_fr if lang == "fr" else _guide_title_en
|
||||
blocks.append("# " + content)
|
||||
for section in root.find_all("section"):
|
||||
_md_block(section, blocks, loc)
|
||||
blocks.append(
|
||||
"---\n\n"
|
||||
+ _export_footer(lang)
|
||||
)
|
||||
md = "\n\n".join(b for b in blocks if b.strip()) + "\n"
|
||||
return md.encode("utf-8")
|
||||
|
||||
|
||||
_guide_title_fr = "Guide d'utilisation ObsiGate"
|
||||
_guide_title_en = "ObsiGate User Guide"
|
||||
|
||||
|
||||
def _export_footer(lang: str) -> str:
|
||||
loc = _locale_strings(lang)
|
||||
template = loc.get("guide105.export_footer", "")
|
||||
if "%s" not in template and "{" not in template:
|
||||
template = "ObsiGate {version}"
|
||||
today = datetime.datetime.now(tz=datetime.UTC).date().isoformat()
|
||||
return _collapse(template).format(version=_app_version(), date=today)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML (pour le PDF) — mêmes règles, sortie balisée propre
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _html_inline(node: Node | str, loc: dict[str, str]) -> str:
|
||||
if isinstance(node, str):
|
||||
return html.escape(_collapse(node), quote=False)
|
||||
tag = node.tag
|
||||
kids = _resolve_i18n(node, loc)
|
||||
inner = "".join(_html_inline(c, loc) for c in kids)
|
||||
if tag == "br":
|
||||
return " "
|
||||
if tag in ("strong", "b") and inner.strip():
|
||||
return f"<strong>{inner}</strong>"
|
||||
if tag in ("em",) and inner.strip():
|
||||
return f"<em>{inner}</em>"
|
||||
if tag == "code":
|
||||
t = inner.strip()
|
||||
return f"<code>{t}</code>" if t else ""
|
||||
if tag == "kbd":
|
||||
t = inner.strip()
|
||||
return f"<code>{t}</code>" if t else ""
|
||||
if tag == "a":
|
||||
href = node.attrs.get("href") or ""
|
||||
if href.startswith("http"):
|
||||
return f'<a href="{html.escape(href, quote=True)}">{inner}</a>'
|
||||
return inner
|
||||
return inner
|
||||
|
||||
|
||||
def _html_block(node: Node | str, out: list[str], loc: dict[str, str]) -> None:
|
||||
if isinstance(node, str):
|
||||
t = _collapse(node)
|
||||
if t:
|
||||
out.append(f"<p>{html.escape(t, quote=False)}</p>")
|
||||
return
|
||||
if any(c in node.cls().split() for c in _SKIP_CLASSES):
|
||||
return
|
||||
tag = node.tag
|
||||
|
||||
if tag == "pre":
|
||||
raw = _pre_text(node)
|
||||
classes = _pre_classes(node)
|
||||
if "language-mermaid" in classes:
|
||||
png = diagram_png_for(raw)
|
||||
if png is not None:
|
||||
url = "file:///" + str(png).replace("\\", "/").lstrip("/")
|
||||
out.append(f'<img src="{url}" style="max-width: 100%" />')
|
||||
return
|
||||
out.append(f"<pre><code>{html.escape(raw, quote=False)}</code></pre>")
|
||||
return
|
||||
|
||||
kids = _resolve_i18n(node, loc)
|
||||
|
||||
if tag in ("h2", "h3", "h4"):
|
||||
text = _collapse("".join(_html_inline(c, loc) for c in kids))
|
||||
if text:
|
||||
out.append(f"<{tag}>{text}</{tag}>")
|
||||
return
|
||||
|
||||
if tag == "p":
|
||||
text = "".join(_html_inline(c, loc) for c in kids).strip()
|
||||
if text:
|
||||
out.append(f"<p>{text}</p>")
|
||||
return
|
||||
|
||||
if tag in ("ul", "ol"):
|
||||
out.append(_html_list(kids, loc, tag))
|
||||
return
|
||||
|
||||
if tag == "table":
|
||||
out.append(_html_table(node, loc))
|
||||
return
|
||||
|
||||
for c in kids:
|
||||
_html_block(c, out, loc)
|
||||
|
||||
|
||||
def _html_list(items: list[Node | str], loc: dict[str, str], kind: str) -> str:
|
||||
parts: list[str] = []
|
||||
n = 0
|
||||
for li in items:
|
||||
if isinstance(li, str):
|
||||
continue
|
||||
if li.tag == "li":
|
||||
n += 1
|
||||
text_parts: list[str] = []
|
||||
nested: list[Node] = []
|
||||
for c in li.children:
|
||||
if isinstance(c, Node) and c.tag in ("ul", "ol"):
|
||||
nested.append(c)
|
||||
else:
|
||||
text_parts.append(_html_inline(c, loc))
|
||||
line = "".join(text_parts).strip()
|
||||
inner = line + "".join(_html_list(s.children, loc, s.tag) for s in nested)
|
||||
if inner:
|
||||
parts.append(f"<li>{inner}</li>")
|
||||
elif li.tag in ("ul", "ol"):
|
||||
parts.append(_html_list(li.children, loc, li.tag))
|
||||
body = "".join(parts)
|
||||
return f"<{kind}>{body}</{kind}>"
|
||||
|
||||
|
||||
def _html_table(node: Node, loc: dict[str, str]) -> str:
|
||||
rows_html: list[str] = []
|
||||
for tr in node.find_all("tr"):
|
||||
cells: list[str] = []
|
||||
for td in tr.children:
|
||||
if isinstance(td, Node) and td.tag in ("td", "th"):
|
||||
tag = td.tag
|
||||
inner = _collapse("".join(_html_inline(c, loc) for c in td.children))
|
||||
cells.append(f"<{tag}>{inner}</{tag}>")
|
||||
if cells:
|
||||
rows_html.append("<tr>{}</tr>".format("".join(cells)))
|
||||
return "<table>{}</table>".format("".join(rows_html))
|
||||
|
||||
|
||||
def build_guide_html(lang: str = "fr") -> str:
|
||||
"""Corps HTML autonome du guide (pour rendu PDF)."""
|
||||
index_html = _read_index_html()
|
||||
loc = _locale_strings(lang)
|
||||
tree = _TreeBuilder()
|
||||
tree.feed(_guide_fragment(index_html))
|
||||
root = tree.root.children[0]
|
||||
assert isinstance(root, Node)
|
||||
|
||||
blocks: list[str] = []
|
||||
for section in root.find_all("section"):
|
||||
_html_block(section, blocks, loc)
|
||||
return "\n".join(blocks)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF (WeasyPrint, repli reportlab)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_guide_pdf(lang: str = "fr") -> bytes:
|
||||
lang_norm = lang if lang in ("fr", "en") else "fr"
|
||||
title = _guide_title_fr if lang_norm == "fr" else _guide_title_en
|
||||
loc = _locale_strings(lang_norm)
|
||||
note = loc.get("guide105.arch_diagram_note", "")
|
||||
footer = _export_footer(lang_norm)
|
||||
try:
|
||||
from backend.pdf_export import build_pdf_html, generate_pdf
|
||||
|
||||
body = build_guide_html(lang_norm)
|
||||
body += (
|
||||
f"<hr><p style='color:#777;font-size:11px'>{html.escape(note, quote=False)} — {html.escape(footer, quote=False)}</p>"
|
||||
)
|
||||
return generate_pdf(build_pdf_html(body, title), title)
|
||||
except Exception as e: # WeasyPrint lève à l'import OU au rendu (GTK absent)
|
||||
logger.warning("WeasyPrint indisponible pour le guide PDF (%s) — repli reportlab", e)
|
||||
md = build_guide_markdown(lang_norm).decode("utf-8")
|
||||
from backend.tools.documents import _render_reportlab_pdf
|
||||
|
||||
return _render_reportlab_pdf(md, title)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Point d'entrée + cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_guide_document(fmt: str, lang: str) -> tuple[bytes, str, str]:
|
||||
"""Retourne (octets, media_type, filename) pour le format demandé.
|
||||
|
||||
``fmt`` : ``md`` | ``pdf``. Résultat mis en cache tant que index.html et
|
||||
fr.json ne changent pas (les locales en ne divergent jamais sur les
|
||||
structures ; la signature couvre l'essentiel).
|
||||
"""
|
||||
fmt = "pdf" if fmt == "pdf" else "md"
|
||||
lang = "en" if lang == "en" else "fr"
|
||||
key = (fmt, lang)
|
||||
sig = _signature()
|
||||
hit = _cache.get(key)
|
||||
if hit and hit[0] == sig:
|
||||
payload = hit[1]
|
||||
else:
|
||||
payload = build_guide_pdf(lang) if fmt == "pdf" else build_guide_markdown(lang)
|
||||
_cache[key] = (sig, payload)
|
||||
fname = f"ObsiGate-Guide-{_app_version()}-{lang}.{fmt}"
|
||||
media = "application/pdf" if fmt == "pdf" else "text/markdown; charset=utf-8"
|
||||
return payload, media, fname
|
||||
@@ -0,0 +1,126 @@
|
||||
# backend/hub_roles.py
|
||||
# Hub HUB-19 (propagation, #228) — Rôles par application : consommation
|
||||
# du registre du Hub par ObsiGate, sur le contrat posé par l'hôte
|
||||
# pilote ShaarIt (0.7.0, décision D3(a) du plan Hub v2).
|
||||
#
|
||||
# À l'ouverture de session draco (validation locale du cookie partagé,
|
||||
# `draco_session.user_for_draco`), ObsiGate demande au Hub le niveau
|
||||
# du porteur pour l'application `obsigate` :
|
||||
# ``GET {hub}/api/roles/obsigate``, en relayant l'en-tête Cookie
|
||||
# complet de la requête, ``Accept: application/json``, délai 2 s.
|
||||
# Réponse 200 : ``{"app", "sub", "role", "explicite",
|
||||
# "ttl_secondes": 60}``, rôle ∈ {aucun, utilisateur, administrateur}.
|
||||
#
|
||||
# Cache mémoire par `sub`, TTL 60 s, sur le patron du pilote (et du
|
||||
# cache de révocations draco) : l'horodatage est posé **avant**
|
||||
# l'appel — un échec conserve l'ancien cache et ne déclenche pas de
|
||||
# rafale de requêtes vers un Hub en panne.
|
||||
#
|
||||
# **Repli obligatoire** : Hub injoignable, en erreur ou réponse
|
||||
# illisible → rôle **inconnu** (``None``) et l'appelant garde son
|
||||
# comportement actuel. Le registre ne crée jamais un verrouillage
|
||||
# nouveau.
|
||||
#
|
||||
# URL du Hub : `draco_config.app_url("hub")` — la même résolution que
|
||||
# la barre commune (#223) : variable explicite `DRACO_HUB_URL` d'abord,
|
||||
# dérivation par l'hôte sœur (suffixe d'environnement de l'hôte de
|
||||
# l'agent) ensuite, défaut officiel enfin. Jamais d'URL dev figée.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger("obsigate.hub_roles")
|
||||
|
||||
#: Identifiant d'ObsiGate dans le registre des applications du Hub.
|
||||
APP = "obsigate"
|
||||
|
||||
#: Niveaux reconnus du registre (contrat D3(a)).
|
||||
ROLES = ("aucun", "utilisateur", "administrateur")
|
||||
|
||||
#: TTL du cache, aligné sur ``ttl_secondes`` annoncé par le Hub (60 s).
|
||||
CACHE_TTL_SECONDS = 60
|
||||
|
||||
#: Délai maximal de la consultation du Hub (contrat : 2 s).
|
||||
DELAI_SECONDS = 2
|
||||
|
||||
_lock = threading.Lock()
|
||||
_cache: dict[str, str] = {} # sub -> rôle connu
|
||||
_cache_at: float | None = None # horodatage (monotone) du dernier appel
|
||||
|
||||
|
||||
def _hub_url() -> str:
|
||||
"""Base publique du Hub, résolue comme pour la barre commune."""
|
||||
from backend.draco_config import app_url
|
||||
|
||||
return app_url("hub")
|
||||
|
||||
|
||||
def _consulter_hub(url: str, cookie: str) -> str | None:
|
||||
"""Appelle le Hub et retourne le rôle validé, ou ``None``.
|
||||
|
||||
Toute erreur (réseau, délai dépassé, statut ≠ 200, JSON illisible,
|
||||
rôle hors contrat) retourne ``None`` : rôle inconnu, repli.
|
||||
``cookie`` est l'en-tête Cookie complet de la requête, relayé tel
|
||||
quel — c'est lui qui prouve l'identité du porteur au Hub.
|
||||
"""
|
||||
requete = urllib.request.Request(
|
||||
url,
|
||||
method="GET",
|
||||
headers={"Cookie": cookie, "Accept": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(requete, timeout=DELAI_SECONDS) as reponse:
|
||||
if reponse.status != 200:
|
||||
return None
|
||||
donnees = json.loads(reponse.read().decode("utf-8"))
|
||||
except Exception: # le registre ne casse jamais une ouverture de session
|
||||
return None
|
||||
if not isinstance(donnees, dict):
|
||||
return None
|
||||
role = donnees.get("role")
|
||||
return role if role in ROLES else None
|
||||
|
||||
|
||||
def role_pour(sub: str, cookie: str) -> str | None:
|
||||
"""Niveau connu pour ce `sub` : un des ``ROLES`` si le Hub a
|
||||
répondu, ``None`` si le rôle est inconnu (Hub injoignable/erreur)
|
||||
— repli = comportement actuel de l'appelant.
|
||||
|
||||
`cookie` est l'en-tête Cookie complet de la requête d'origine,
|
||||
relayé tel quel au Hub.
|
||||
"""
|
||||
global _cache_at
|
||||
if not sub:
|
||||
return None
|
||||
with _lock:
|
||||
if _cache_at is not None \
|
||||
and time.monotonic() - _cache_at < CACHE_TTL_SECONDS \
|
||||
and sub in _cache:
|
||||
return _cache[sub]
|
||||
# Horodatage posé AVANT l'appel (patron du pilote) : l'ancien
|
||||
# cache reste en vigueur, un échec ne déclenche pas de rafale.
|
||||
_cache_at = time.monotonic()
|
||||
url = f"{_hub_url().rstrip('/')}/api/roles/{APP}"
|
||||
role = _consulter_hub(url, cookie)
|
||||
if role is None:
|
||||
logger.warning(
|
||||
"Registre des rôles du Hub indisponible (%s) — "
|
||||
"comportement actuel conservé", url)
|
||||
return None
|
||||
with _lock:
|
||||
_cache[sub] = role
|
||||
_cache_at = time.monotonic()
|
||||
return role
|
||||
|
||||
|
||||
def reinitialiser_cache() -> None:
|
||||
"""Vide le cache des rôles (tests, redémarrage à chaud)."""
|
||||
global _cache_at
|
||||
with _lock:
|
||||
_cache.clear()
|
||||
_cache_at = None
|
||||
+713
-101
File diff suppressed because it is too large
Load Diff
+635
-4040
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
"""Signed, single-use confirmation tokens for MCP mutations (Phase E4).
|
||||
|
||||
MCP has no "Apply" button, so mutating tools are exposed in two steps:
|
||||
``propose_<tool>`` returns a preview plus a **signed token**, and
|
||||
``apply_<tool>`` consumes that token to execute the mutation.
|
||||
|
||||
The token is a short-lived JWT (same secret as the app) carrying the tool name
|
||||
and its arguments. Single use is enforced by a persisted JTI blacklist, which
|
||||
also protects against replay and TOCTOU (a stale proposal cannot be re-applied).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from jose import JWTError, jwt
|
||||
|
||||
from backend.auth.jwt_handler import get_secret_key
|
||||
|
||||
logger = logging.getLogger("obsigate.mcp.confirmations")
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
TOKEN_TYPE = "mcp_confirmation"
|
||||
|
||||
# Default token lifetime (seconds); override with OBSIGATE_MCP_CONFIRMATION_TTL.
|
||||
DEFAULT_TTL = int(os.environ.get("OBSIGATE_MCP_CONFIRMATION_TTL", "300"))
|
||||
|
||||
_USED_TOKENS_FILE = Path("data/mcp_used_tokens.json")
|
||||
_used_lock = threading.RLock()
|
||||
_used_loaded = False
|
||||
_used_jtis: dict[str, int] = {}
|
||||
|
||||
|
||||
class ConfirmationError(Exception):
|
||||
"""Raised when a confirmation token is invalid, expired, or already used."""
|
||||
|
||||
def __init__(self, message: str, *, code: str = "invalid_confirmation"):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
|
||||
|
||||
def _load_used() -> None:
|
||||
"""Load used JTIs from disk once, dropping expired entries."""
|
||||
global _used_loaded, _used_jtis
|
||||
if _used_loaded:
|
||||
return
|
||||
with _used_lock:
|
||||
if _used_loaded:
|
||||
return
|
||||
if _USED_TOKENS_FILE.exists():
|
||||
try:
|
||||
data = json.loads(_USED_TOKENS_FILE.read_text(encoding="utf-8"))
|
||||
now = int(time.time())
|
||||
_used_jtis = {jti: exp for jti, exp in data.items() if int(exp) > now}
|
||||
except Exception as e: # pragma: no cover - corrupt store
|
||||
logger.warning(f"Failed to load used MCP tokens: {e}")
|
||||
_used_jtis = {}
|
||||
_used_loaded = True
|
||||
|
||||
|
||||
def _save_used() -> None:
|
||||
"""Persist used JTIs with their expiry (best-effort)."""
|
||||
try:
|
||||
_USED_TOKENS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = _USED_TOKENS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(_used_jtis), encoding="utf-8")
|
||||
tmp.replace(_USED_TOKENS_FILE)
|
||||
except Exception as e: # pragma: no cover - disk error
|
||||
logger.warning(f"Failed to persist used MCP tokens: {e}")
|
||||
|
||||
|
||||
def create_confirmation_token(
|
||||
tool: str,
|
||||
arguments: dict[str, Any],
|
||||
username: str,
|
||||
*,
|
||||
ttl: int | None = None,
|
||||
) -> str:
|
||||
"""Create a signed confirmation token for a pending mutation."""
|
||||
now = int(time.time())
|
||||
payload = {
|
||||
"type": TOKEN_TYPE,
|
||||
"tool": tool,
|
||||
"arguments": arguments,
|
||||
"sub": username,
|
||||
"jti": str(uuid.uuid4()),
|
||||
"iat": now,
|
||||
"exp": now + (ttl if ttl is not None else DEFAULT_TTL),
|
||||
}
|
||||
return jwt.encode(payload, get_secret_key(), algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def peek_confirmation_token(token: str) -> dict[str, Any]:
|
||||
"""Decode and validate a token without consuming it (signature + expiry)."""
|
||||
try:
|
||||
payload = jwt.decode(token, get_secret_key(), algorithms=[ALGORITHM])
|
||||
except JWTError as e:
|
||||
raise ConfirmationError("Invalid or expired confirmation token", code="invalid_confirmation") from e
|
||||
|
||||
if payload.get("type") != TOKEN_TYPE:
|
||||
raise ConfirmationError("Wrong token type", code="invalid_confirmation")
|
||||
if not payload.get("tool") or not payload.get("sub"):
|
||||
raise ConfirmationError("Malformed confirmation token", code="invalid_confirmation")
|
||||
return payload
|
||||
|
||||
|
||||
def consume_confirmation_token(token: str, username: str) -> dict[str, Any]:
|
||||
"""Validate a token, enforce single use, and return its payload.
|
||||
|
||||
Raises:
|
||||
ConfirmationError: invalid/expired token, wrong user, or replay.
|
||||
"""
|
||||
payload = peek_confirmation_token(token)
|
||||
|
||||
if payload.get("sub") != username:
|
||||
raise ConfirmationError("Confirmation token does not belong to this user", code="forbidden")
|
||||
|
||||
jti = payload.get("jti")
|
||||
if not jti:
|
||||
raise ConfirmationError("Malformed confirmation token", code="invalid_confirmation")
|
||||
|
||||
_load_used()
|
||||
now = int(time.time())
|
||||
with _used_lock:
|
||||
if jti in _used_jtis:
|
||||
raise ConfirmationError("Confirmation token already used", code="token_reused")
|
||||
# Record as used *before* returning so a concurrent replay is rejected.
|
||||
_used_jtis[jti] = int(payload.get("exp", now))
|
||||
# Opportunistic cleanup of expired JTIs.
|
||||
for old_jti in [k for k, exp in _used_jtis.items() if exp <= now]:
|
||||
_used_jtis.pop(old_jti, None)
|
||||
_save_used()
|
||||
|
||||
return payload
|
||||
@@ -0,0 +1,505 @@
|
||||
"""ObsiGate MCP server — Streamable HTTP transport (Phase E).
|
||||
|
||||
Exposes the shared AI tool layer (``backend.tools``) to external MCP clients
|
||||
(Claude Desktop, Cursor…). The same registry that powers the in-app assistant
|
||||
is registered here, so the two fronts never diverge.
|
||||
|
||||
Primitives:
|
||||
- **Tools** — read/search tools directly; mutating tools as a two-step
|
||||
``propose_<tool>`` / ``apply_<tool>`` pair (signed, single-use token).
|
||||
- **Resources** — accessible vaults (``vault://<name>``) and files
|
||||
(``vault://<name>/<path>``), read-only and secret-redacted.
|
||||
- **Prompts** — reusable note/summary templates.
|
||||
|
||||
Transport: Streamable HTTP mounted at ``/mcp`` (auth ``Authorization: Bearer
|
||||
<JWT>``). ``stdio`` is left for later.
|
||||
|
||||
The transport manager is started lazily on the first request so the endpoint
|
||||
also works in tests (where the ASGI lifespan is not run).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import difflib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
from mcp import types
|
||||
from mcp.server.lowlevel import Server
|
||||
from mcp.server.lowlevel.helper_types import ReadResourceContents
|
||||
from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
|
||||
from pydantic import AnyUrl
|
||||
from starlette._utils import get_route_path
|
||||
from starlette.requests import Request
|
||||
from starlette.routing import BaseRoute, Match
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
from backend.auth.middleware import get_current_user, is_auth_enabled
|
||||
from backend.services.files import read_file_text
|
||||
from backend.services.vaults import list_accessible_vaults
|
||||
from backend.tools.api import (
|
||||
FILES_SCOPE_TOOLS,
|
||||
ToolContext,
|
||||
ToolError,
|
||||
ToolMode,
|
||||
ToolRisk,
|
||||
ToolScope,
|
||||
call_tool,
|
||||
get_tool,
|
||||
list_tools,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.mcp")
|
||||
|
||||
SERVER_NAME = "obsigate"
|
||||
SERVER_INSTRUCTIONS = (
|
||||
"ObsiGate exposes your Obsidian vaults: read, search and (with confirmation) "
|
||||
"create, edit, rename, move or delete notes. Mutating tools require the "
|
||||
"two-step propose_/apply_ flow."
|
||||
)
|
||||
|
||||
# Cap on the file size returned by resources (bytes).
|
||||
MAX_RESOURCE_BYTES = 200_000
|
||||
|
||||
|
||||
def _anonymous_user() -> dict[str, Any]:
|
||||
return {
|
||||
"username": "anonymous",
|
||||
"display_name": "Anonymous",
|
||||
"role": "admin",
|
||||
"vaults": ["*"],
|
||||
"active": True,
|
||||
"_token_vaults": ["*"],
|
||||
}
|
||||
|
||||
|
||||
def _authenticate(request: Request) -> dict[str, Any] | None:
|
||||
"""Resolve the caller from the ``Authorization: Bearer`` header."""
|
||||
if not is_auth_enabled():
|
||||
return _anonymous_user()
|
||||
|
||||
from fastapi.security import HTTPAuthorizationCredentials
|
||||
|
||||
header = request.headers.get("authorization", "")
|
||||
if not header.lower().startswith("bearer "):
|
||||
return None
|
||||
credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials=header[7:].strip())
|
||||
return get_current_user(request, credentials)
|
||||
|
||||
|
||||
def _user_from_context(server: Server) -> dict[str, Any]:
|
||||
"""Return the authenticated user attached to the current request scope."""
|
||||
request = server.request_context.request
|
||||
if request is None: # pragma: no cover - stdio not supported yet
|
||||
raise ToolError("No request context available", code="no_request_context")
|
||||
user = getattr(request.state, "user", None)
|
||||
if not user:
|
||||
raise ToolError("Unauthenticated MCP request", code="unauthenticated")
|
||||
return user
|
||||
|
||||
|
||||
def _text(payload: Any) -> list[types.Content]:
|
||||
"""Serialize a payload as a single text content block."""
|
||||
return [types.TextContent(type="text", text=json.dumps(payload, ensure_ascii=False, default=str))]
|
||||
|
||||
|
||||
def _unified_diff(old: str, new: str) -> str:
|
||||
diff = difflib.unified_diff(
|
||||
old.splitlines(keepends=True),
|
||||
new.splitlines(keepends=True),
|
||||
fromfile="current",
|
||||
tofile="proposed",
|
||||
)
|
||||
return "".join(diff)
|
||||
|
||||
|
||||
def _build_preview(spec: Any, params: Any) -> dict[str, Any]:
|
||||
"""Build a human-readable preview for a proposed mutation."""
|
||||
arguments = params.model_dump()
|
||||
preview: dict[str, Any] = {"tool": spec.name, "arguments": arguments}
|
||||
|
||||
if spec.name in ("create_file", "edit_file", "append_to_file"):
|
||||
vault = arguments.get("vault")
|
||||
path = arguments.get("path", "")
|
||||
content = arguments.get("content", "")
|
||||
try:
|
||||
current = read_file_text(vault, path, redact=True, max_bytes=MAX_RESOURCE_BYTES)["content"]
|
||||
except Exception:
|
||||
current = ""
|
||||
if spec.name == "append_to_file":
|
||||
separator = "" if (not current or current.endswith("\n")) else "\n"
|
||||
proposed = current + separator + content
|
||||
else:
|
||||
proposed = content
|
||||
preview["diff"] = _unified_diff(current, proposed)
|
||||
|
||||
return preview
|
||||
|
||||
|
||||
def _tool_definitions() -> list[types.Tool]:
|
||||
"""Build the MCP tool list from the shared registry."""
|
||||
definitions: list[types.Tool] = []
|
||||
for spec in list_tools(scope=ToolScope.MCP):
|
||||
if spec.risk == ToolRisk.READ:
|
||||
definitions.append(
|
||||
types.Tool(
|
||||
name=spec.name,
|
||||
description=spec.description,
|
||||
inputSchema=spec.parameters_schema(),
|
||||
annotations=types.ToolAnnotations(readOnlyHint=True, openWorldHint=False),
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
destructive = spec.risk == ToolRisk.DANGEROUS
|
||||
definitions.append(
|
||||
types.Tool(
|
||||
name=f"propose_{spec.name}",
|
||||
description=(
|
||||
f"Propose to run '{spec.name}' and return a confirmation token. "
|
||||
"No change is made until apply_ is called."
|
||||
),
|
||||
inputSchema=spec.parameters_schema(),
|
||||
annotations=types.ToolAnnotations(
|
||||
readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False
|
||||
),
|
||||
)
|
||||
)
|
||||
definitions.append(
|
||||
types.Tool(
|
||||
name=f"apply_{spec.name}",
|
||||
description=f"Apply a previously proposed '{spec.name}' using its confirmation token.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"confirmation_token": {
|
||||
"type": "string",
|
||||
"description": f"Token returned by propose_{spec.name}",
|
||||
}
|
||||
},
|
||||
"required": ["confirmation_token"],
|
||||
},
|
||||
annotations=types.ToolAnnotations(
|
||||
readOnlyHint=False, destructiveHint=destructive, idempotentHint=False, openWorldHint=False
|
||||
),
|
||||
)
|
||||
)
|
||||
return definitions
|
||||
|
||||
|
||||
def _mcp_base_name(mcp_name: str) -> str:
|
||||
"""Strip the ``propose_``/``apply_`` prefix back to the registry tool name."""
|
||||
for prefix in ("propose_", "apply_"):
|
||||
if mcp_name.startswith(prefix):
|
||||
return mcp_name[len(prefix):]
|
||||
return mcp_name
|
||||
|
||||
|
||||
def _parse_vault_uri(uri: str) -> tuple[str, str]:
|
||||
"""Split ``vault://name/path`` into ``(vault, path)``."""
|
||||
parsed = urlparse(uri)
|
||||
if parsed.scheme != "vault" or not parsed.netloc:
|
||||
raise ValueError(f"Unsupported resource URI: {uri}")
|
||||
return parsed.netloc, parsed.path.lstrip("/")
|
||||
|
||||
|
||||
def build_server() -> Server:
|
||||
"""Create and configure the low-level MCP server."""
|
||||
from backend.version import get_version
|
||||
|
||||
server: Server = Server(SERVER_NAME, version=get_version(), instructions=SERVER_INSTRUCTIONS)
|
||||
|
||||
# ── Tools ──────────────────────────────────────────────────────────
|
||||
@server.list_tools()
|
||||
async def _list_tools() -> list[types.Tool]:
|
||||
# #199 : un token à portée "files" ne voit que les outils fichiers.
|
||||
user = _user_from_context(server)
|
||||
if (user.get("_token_scope") or "full") == "files":
|
||||
return [t for t in _tool_definitions() if _mcp_base_name(t.name) in FILES_SCOPE_TOOLS]
|
||||
return _tool_definitions()
|
||||
|
||||
@server.call_tool()
|
||||
async def _call_tool(name: str, arguments: dict[str, Any]) -> list[types.Content]:
|
||||
user = _user_from_context(server)
|
||||
|
||||
if name.startswith("propose_"):
|
||||
return _propose(user, name[len("propose_"):], arguments)
|
||||
if name.startswith("apply_"):
|
||||
return _apply(user, name[len("apply_"):], arguments)
|
||||
|
||||
spec = get_tool(name)
|
||||
if spec is None:
|
||||
raise ValueError(f"Unknown tool: {name}")
|
||||
if spec.risk != ToolRisk.READ:
|
||||
raise ValueError(f"Tool '{name}' is mutating; use propose_{name}/apply_{name}")
|
||||
|
||||
ctx = ToolContext(user=user, mode=ToolMode.MCP)
|
||||
try:
|
||||
result = call_tool(name, ctx, arguments)
|
||||
except ToolError as e:
|
||||
return _text(e.to_dict())
|
||||
return _text({"ok": True, "data": result.data})
|
||||
|
||||
# ── Resources ──────────────────────────────────────────────────────
|
||||
@server.list_resources()
|
||||
async def _list_resources() -> list[types.Resource]:
|
||||
user = _user_from_context(server)
|
||||
return [
|
||||
types.Resource(
|
||||
uri=AnyUrl(f"vault://{v['name']}"),
|
||||
name=v["name"],
|
||||
description=f"ObsiGate vault '{v['name']}' ({v['file_count']} files)",
|
||||
mimeType="application/x-obsigate-vault",
|
||||
)
|
||||
for v in list_accessible_vaults(user)
|
||||
]
|
||||
|
||||
@server.list_resource_templates()
|
||||
async def _list_resource_templates() -> list[types.ResourceTemplate]:
|
||||
return [
|
||||
types.ResourceTemplate(
|
||||
uriTemplate="vault://{vault}/{path}",
|
||||
name="Vault file",
|
||||
description="Read a text file from a vault (secrets redacted)",
|
||||
mimeType="text/markdown",
|
||||
)
|
||||
]
|
||||
|
||||
@server.read_resource()
|
||||
async def _read_resource(uri: Any) -> list[ReadResourceContents]:
|
||||
user = _user_from_context(server)
|
||||
vault, path = _parse_vault_uri(str(uri))
|
||||
ctx = ToolContext(user=user, mode=ToolMode.MCP)
|
||||
ctx.require_tool_allowed("read_file") # #199 : portée inconnue → refus
|
||||
ctx.require_vault_access(vault)
|
||||
data = read_file_text(vault, path, redact=True, max_bytes=MAX_RESOURCE_BYTES)
|
||||
mime = "text/markdown" if Path(path).suffix.lower() == ".md" else "text/plain"
|
||||
return [ReadResourceContents(content=data["content"], mime_type=mime)]
|
||||
|
||||
# ── Prompts ────────────────────────────────────────────────────────
|
||||
@server.list_prompts()
|
||||
async def _list_prompts() -> list[types.Prompt]:
|
||||
return [
|
||||
types.Prompt(
|
||||
name="summarize-directory",
|
||||
description="Summarize every note in a vault directory.",
|
||||
arguments=[
|
||||
types.PromptArgument(name="vault", description="Vault name", required=True),
|
||||
types.PromptArgument(name="path", description="Directory path (empty = root)", required=False),
|
||||
],
|
||||
),
|
||||
types.Prompt(
|
||||
name="generate-note",
|
||||
description="Draft a new note on a topic, using the vault for context.",
|
||||
arguments=[
|
||||
types.PromptArgument(name="topic", description="Note topic", required=True),
|
||||
types.PromptArgument(name="vault", description="Target vault name", required=True),
|
||||
],
|
||||
),
|
||||
types.Prompt(
|
||||
name="find-related",
|
||||
description="Find notes related to a given note.",
|
||||
arguments=[
|
||||
types.PromptArgument(name="vault", description="Vault name", required=True),
|
||||
types.PromptArgument(name="path", description="Reference note path", required=True),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
@server.get_prompt()
|
||||
async def _get_prompt(name: str, arguments: dict[str, str] | None) -> types.GetPromptResult:
|
||||
args = arguments or {}
|
||||
|
||||
def _require(key: str) -> str:
|
||||
value = (args.get(key) or "").strip()
|
||||
if not value:
|
||||
raise ValueError(f"Missing required prompt argument: {key}")
|
||||
return value
|
||||
|
||||
if name == "summarize-directory":
|
||||
vault = _require("vault")
|
||||
path = (args.get("path") or "").strip()
|
||||
text = (
|
||||
f"List the directory '{path or '/'}' of vault '{vault}' (use list_directory), "
|
||||
"read the notes it contains, then write a concise summary with the key ideas."
|
||||
)
|
||||
elif name == "generate-note":
|
||||
topic = _require("topic")
|
||||
vault = _require("vault")
|
||||
text = (
|
||||
f"Draft a well-structured Markdown note about '{topic}' in vault '{vault}'. "
|
||||
"Search the vault first for existing material, then propose the note content."
|
||||
)
|
||||
elif name == "find-related":
|
||||
vault = _require("vault")
|
||||
path = _require("path")
|
||||
text = (
|
||||
f"Read '{path}' in vault '{vault}', then find related notes via backlinks and "
|
||||
"full-text search. Return a short list with why each note is related."
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown prompt: {name}")
|
||||
|
||||
return types.GetPromptResult(
|
||||
description=f"ObsiGate prompt: {name}",
|
||||
messages=[types.PromptMessage(role="user", content=types.TextContent(type="text", text=text))],
|
||||
)
|
||||
|
||||
return server
|
||||
|
||||
|
||||
def _propose(user: dict[str, Any], tool: str, arguments: dict[str, Any]) -> list[types.Content]:
|
||||
"""Validate a mutation, return a preview and a confirmation token."""
|
||||
from backend.mcp.confirmations import DEFAULT_TTL, create_confirmation_token
|
||||
|
||||
spec = get_tool(tool)
|
||||
if spec is None:
|
||||
raise ValueError(f"Unknown tool: {tool}")
|
||||
if spec.risk == ToolRisk.READ:
|
||||
raise ValueError(f"Tool '{tool}' is read-only; call it directly")
|
||||
|
||||
params = spec.input_model.model_validate(arguments)
|
||||
vault = getattr(params, "vault", None)
|
||||
ctx = ToolContext(user=user, mode=ToolMode.MCP)
|
||||
# #199 : les tokens à portée limitée ne peuvent proposer que les outils
|
||||
# fichiers (même garde que call_tool, qui couvre apply_).
|
||||
try:
|
||||
ctx.require_tool_allowed(tool)
|
||||
except ToolError as e:
|
||||
return _text(e.to_dict())
|
||||
if vault and vault != "all":
|
||||
ctx.require_vault_access(vault)
|
||||
if spec.risk == ToolRisk.DANGEROUS:
|
||||
ctx.require_destructive_allowed(vault)
|
||||
|
||||
token = create_confirmation_token(tool, arguments, ctx.username)
|
||||
payload = _build_preview(spec, params)
|
||||
payload["confirmation_token"] = token
|
||||
payload["expires_in"] = DEFAULT_TTL
|
||||
return _text(payload)
|
||||
|
||||
|
||||
def _apply(user: dict[str, Any], tool: str, arguments: dict[str, Any]) -> list[types.Content]:
|
||||
"""Consume a confirmation token and execute the mutation."""
|
||||
from backend.mcp.confirmations import ConfirmationError, consume_confirmation_token
|
||||
|
||||
token = (arguments or {}).get("confirmation_token")
|
||||
if not token:
|
||||
raise ValueError("Missing 'confirmation_token'")
|
||||
|
||||
try:
|
||||
payload = consume_confirmation_token(token, user.get("username", ""))
|
||||
except ConfirmationError as e:
|
||||
return _text({"ok": False, "error": {"code": e.code, "message": e.message}})
|
||||
|
||||
if payload.get("tool") != tool:
|
||||
return _text(
|
||||
{
|
||||
"ok": False,
|
||||
"error": {
|
||||
"code": "token_mismatch",
|
||||
"message": f"Token was issued for '{payload.get('tool')}', not '{tool}'",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
ctx = ToolContext(user=user, mode=ToolMode.MCP, confirmed=True)
|
||||
try:
|
||||
result = call_tool(tool, ctx, payload.get("arguments") or {}, confirm=True)
|
||||
except ToolError as e:
|
||||
return _text(e.to_dict())
|
||||
return _text({"ok": True, "data": result.data})
|
||||
|
||||
|
||||
class McpASGIApp:
|
||||
"""ASGI wrapper: authenticates the request, then runs the MCP transport.
|
||||
|
||||
The session manager is started lazily on first use and kept alive for the
|
||||
process lifetime, so the endpoint works both under uvicorn (with lifespan)
|
||||
and under the test client (without entering the ASGI lifespan).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._manager: StreamableHTTPSessionManager | None = None
|
||||
self._start_lock: asyncio.Lock | None = None
|
||||
self._run_task: asyncio.Task | None = None
|
||||
|
||||
def _get_manager(self) -> StreamableHTTPSessionManager:
|
||||
if self._manager is None:
|
||||
self._manager = StreamableHTTPSessionManager(
|
||||
app=build_server(),
|
||||
json_response=True,
|
||||
stateless=False,
|
||||
)
|
||||
return self._manager
|
||||
|
||||
async def _ensure_started(self) -> StreamableHTTPSessionManager:
|
||||
manager = self._get_manager()
|
||||
if getattr(manager, "_task_group", None) is not None:
|
||||
return manager
|
||||
if self._start_lock is None:
|
||||
self._start_lock = asyncio.Lock()
|
||||
async with self._start_lock:
|
||||
if getattr(manager, "_task_group", None) is None:
|
||||
self._run_task = asyncio.create_task(self._run_manager(manager))
|
||||
for _ in range(500):
|
||||
if getattr(manager, "_task_group", None) is not None:
|
||||
break
|
||||
await asyncio.sleep(0.005)
|
||||
return manager
|
||||
|
||||
async def _run_manager(self, manager: StreamableHTTPSessionManager) -> None:
|
||||
async with manager.run():
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
|
||||
request = Request(scope, receive)
|
||||
user = _authenticate(request)
|
||||
if user is None:
|
||||
response = JSONResponse(
|
||||
{"detail": "Authentification requise"},
|
||||
status_code=401,
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
await response(scope, receive, send)
|
||||
return
|
||||
|
||||
scope.setdefault("state", {})["user"] = user
|
||||
manager = await self._ensure_started()
|
||||
await manager.handle_request(scope, receive, send)
|
||||
|
||||
|
||||
# Module-level singleton mounted by ``backend.main`` at ``/mcp``.
|
||||
mcp_app = McpASGIApp()
|
||||
|
||||
|
||||
class McpMount(BaseRoute):
|
||||
"""ASGI route matching ``/mcp`` and ``/mcp/...`` (unlike Starlette's Mount).
|
||||
|
||||
Starlette's :class:`~starlette.routing.Mount` compiles ``/mcp/{path:path}``
|
||||
and therefore does **not** match the bare ``/mcp`` path used by MCP clients.
|
||||
This route matches both forms and delegates to :data:`mcp_app` unchanged.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, path: str = "/mcp") -> None:
|
||||
self.app = app
|
||||
self.path = path.rstrip("/")
|
||||
|
||||
def matches(self, scope: Scope) -> tuple[Match, Scope]:
|
||||
if scope["type"] == "http":
|
||||
route_path = get_route_path(scope)
|
||||
if route_path == self.path or route_path.startswith(self.path + "/"):
|
||||
return Match.FULL, {"endpoint": self.app}
|
||||
return Match.NONE, {}
|
||||
|
||||
async def handle(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Image thumbnail generation and disk cache (roadmap #108-C).
|
||||
|
||||
Thumbnails are generated on demand with Pillow and cached under
|
||||
``<OBSIGATE_DATA_DIR>/.obsigate-cache/thumbs/<sha1>.webp``. The cache key
|
||||
embeds the source path, mtime (ns) and size, so an edited image naturally
|
||||
invalidates its stale thumbnail without any explicit cleanup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_THUMB_SIZE = 256
|
||||
|
||||
# Extensions Pillow cannot decode without extra native libraries: served as-is.
|
||||
_UNDECODABLE = {".svg"}
|
||||
|
||||
|
||||
def thumbs_cache_dir() -> Path:
|
||||
"""Return (and create) the thumbnail cache directory."""
|
||||
base = Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / ".obsigate-cache" / "thumbs"
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
return base
|
||||
|
||||
|
||||
def thumb_cache_path(file_path: Path, size: int) -> Path:
|
||||
"""Compute the deterministic cache path for *file_path* at *size*."""
|
||||
try:
|
||||
st = file_path.stat()
|
||||
stamp = f"{st.st_mtime_ns}:{st.st_size}"
|
||||
except OSError:
|
||||
stamp = "0:0"
|
||||
# Clé de cache miniature (pas un usage sécurité).
|
||||
key = hashlib.sha1(f"{file_path}:{stamp}:{size}".encode()).hexdigest() # nosec B324
|
||||
return thumbs_cache_dir() / f"{key}.webp"
|
||||
|
||||
|
||||
def is_decodable(file_path: Path) -> bool:
|
||||
"""True when Pillow can be expected to decode *file_path*."""
|
||||
return file_path.suffix.lower() not in _UNDECODABLE
|
||||
|
||||
|
||||
def generate_thumbnail(file_path: Path, size: int = DEFAULT_THUMB_SIZE) -> Path | None:
|
||||
"""Generate (or reuse) a WebP thumbnail and return its path.
|
||||
|
||||
Returns ``None`` when the file cannot be decoded (e.g. SVG) or Pillow is
|
||||
unavailable, so the caller can fall back to serving the original.
|
||||
"""
|
||||
cache_path = thumb_cache_path(file_path, size)
|
||||
if cache_path.exists():
|
||||
return cache_path
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
except Exception: # pragma: no cover - Pillow is an optional runtime dep
|
||||
return None
|
||||
|
||||
try:
|
||||
with Image.open(file_path) as opened:
|
||||
# Animated formats: keep only the first frame.
|
||||
if getattr(opened, "is_animated", False):
|
||||
opened.seek(0)
|
||||
img = ImageOps.exif_transpose(opened) or opened
|
||||
if img.mode not in ("RGB", "RGBA"):
|
||||
img = img.convert("RGBA")
|
||||
img.thumbnail((size, size))
|
||||
|
||||
tmp = cache_path.with_suffix(".tmp")
|
||||
img.save(tmp, "WEBP", quality=80)
|
||||
os.replace(tmp, cache_path)
|
||||
return cache_path
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared media type constants and helpers.
|
||||
|
||||
Single source of truth for the file extensions and MIME types handled by the
|
||||
image support (roadmap #108) and reused by the audio/video players (#109).
|
||||
Keeping these sets here avoids the previous duplication (``indexer.py``,
|
||||
``attachment_indexer.py`` and ``main.py`` each carried their own copy).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import mimetypes
|
||||
|
||||
# Image extensions viewable in the browser (HEIC/HEIF deliberately excluded —
|
||||
# no browser decodes them natively; see roadmap #108).
|
||||
IMAGE_EXTENSIONS: frozenset[str] = frozenset({
|
||||
".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".bmp", ".ico",
|
||||
})
|
||||
|
||||
# Audio extensions (socle for #109, not wired into the index yet).
|
||||
AUDIO_EXTENSIONS: frozenset[str] = frozenset({
|
||||
".mp3", ".m4a", ".aac", ".wav", ".ogg", ".oga", ".opus", ".flac",
|
||||
})
|
||||
|
||||
# Video extensions (socle for #109, not wired into the index yet).
|
||||
VIDEO_EXTENSIONS: frozenset[str] = frozenset({
|
||||
".mp4", ".webm", ".mov", ".m4v",
|
||||
})
|
||||
|
||||
MEDIA_EXTENSIONS: frozenset[str] = IMAGE_EXTENSIONS | AUDIO_EXTENSIONS | VIDEO_EXTENSIONS
|
||||
|
||||
# Explicit MIME types for extensions ``mimetypes`` gets wrong or does not know.
|
||||
_MIME_OVERRIDES: dict[str, str] = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".svg": "image/svg+xml",
|
||||
".ico": "image/x-icon",
|
||||
".webp": "image/webp",
|
||||
".m4a": "audio/mp4",
|
||||
".oga": "audio/ogg",
|
||||
".opus": "audio/ogg",
|
||||
".mov": "video/quicktime",
|
||||
".m4v": "video/mp4",
|
||||
}
|
||||
|
||||
|
||||
def is_image(ext: str) -> bool:
|
||||
"""Return True when *ext* (with leading dot, any case) is an image."""
|
||||
return ext.lower() in IMAGE_EXTENSIONS
|
||||
|
||||
|
||||
def is_audio(ext: str) -> bool:
|
||||
"""Return True when *ext* is an audio extension."""
|
||||
return ext.lower() in AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
def is_video(ext: str) -> bool:
|
||||
"""Return True when *ext* is a video extension."""
|
||||
return ext.lower() in VIDEO_EXTENSIONS
|
||||
|
||||
|
||||
def is_media(ext: str) -> bool:
|
||||
"""Return True when *ext* is any supported image/audio/video extension."""
|
||||
return ext.lower() in MEDIA_EXTENSIONS
|
||||
|
||||
|
||||
def media_mime_type(path: str) -> str:
|
||||
"""Return the best MIME type for *path* (extension based).
|
||||
|
||||
Falls back to ``application/octet-stream`` when the type is unknown.
|
||||
"""
|
||||
lower = path.lower()
|
||||
for ext, mime in _MIME_OVERRIDES.items():
|
||||
if lower.endswith(ext):
|
||||
return mime
|
||||
guessed, _ = mimetypes.guess_type(path)
|
||||
return guessed or "application/octet-stream"
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Model-capability metadata for the AI assistant.
|
||||
|
||||
Two layers, in order of trust:
|
||||
|
||||
1. **Provider-declared** (:mod:`backend.provider_capabilities`) — when the
|
||||
provider publishes per-model capabilities in its models endpoint (Mistral
|
||||
``capabilities``, OpenRouter ``architecture``), that declaration *wins* for
|
||||
every flag it mentions. The snapshot is cached by the model-list endpoint.
|
||||
2. **Curated table** (this module) — a static, hand-maintained map of known
|
||||
model-name patterns to capability flags with per-provider defaults. It fills
|
||||
the flags the provider stays silent about, and is the only source for
|
||||
providers and models that declare nothing (offline, no API key, DeepSeek,
|
||||
NVIDIA, QwenCloud, Xiaomi…).
|
||||
|
||||
The UI uses the result to show, when a model is selected, which features it
|
||||
supports (Chat, Embeddings, Rerank, Images, Video, Audio Speech, Audio
|
||||
Transcriptions, Vision).
|
||||
|
||||
The curated layer is intentionally conservative: an unknown model falls back to
|
||||
the provider default (usually ``chat`` only), so we never claim a capability the
|
||||
model may not have. A provider declaration, on the other hand, is authoritative
|
||||
in both directions — it can also *revoke* a flag the curated table guessed
|
||||
wrongly (e.g. ``mistral-embed`` declares no chat).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.provider_capabilities import get_declared_capabilities
|
||||
|
||||
# Ordered list of capability keys exposed to the UI. Keep in sync with the
|
||||
# frontend ``AI_CAPABILITY_KEYS`` and the i18n ``ai.cap_*`` labels.
|
||||
CAPABILITY_KEYS: tuple[str, ...] = (
|
||||
"chat",
|
||||
"embeddings",
|
||||
"rerank",
|
||||
"images",
|
||||
"video",
|
||||
"audio_speech",
|
||||
"audio_transcription",
|
||||
"vision",
|
||||
)
|
||||
|
||||
|
||||
def _caps(**kwargs: bool) -> dict[str, bool]:
|
||||
"""Build a full capability dict (missing flags default to False)."""
|
||||
base = {key: False for key in CAPABILITY_KEYS}
|
||||
base.update(kwargs)
|
||||
return base
|
||||
|
||||
|
||||
# Provider defaults, used when no model-name pattern matches.
|
||||
_PROVIDER_DEFAULTS: dict[str, dict[str, bool]] = {
|
||||
"deepseek": _caps(chat=True),
|
||||
"openrouter": _caps(chat=True),
|
||||
"gemini": _caps(chat=True, vision=True, embeddings=True, audio_speech=True),
|
||||
"ollama": _caps(chat=True, embeddings=True),
|
||||
"nvidia": _caps(chat=True),
|
||||
"qwencloud": _caps(chat=True),
|
||||
"xiaomi": _caps(chat=True),
|
||||
# Mistral: chat only. ``embeddings`` used to be assumed for every Mistral
|
||||
# model, which wrongly labelled mistral-large / codestral as embedders
|
||||
# (BUG-044); the ``embed`` rule below covers the real embedding models.
|
||||
"mistral": _caps(chat=True),
|
||||
}
|
||||
|
||||
# Ordered (substrings, capabilities) rules — the first matching rule wins.
|
||||
# More specific modalities are listed before the broad vision/chat rule.
|
||||
_MODEL_RULES: list[tuple[tuple[str, ...], dict[str, bool]]] = [
|
||||
# Rerankers.
|
||||
(("rerank", "cross-encoder"), _caps(rerank=True)),
|
||||
# Embedding models (usually not chat-capable).
|
||||
(
|
||||
("text-embedding", "embed", "bge-", "e5-", "nomic-embed", "gte-"),
|
||||
_caps(embeddings=True),
|
||||
),
|
||||
# Speech-to-text / transcription.
|
||||
(
|
||||
("whisper", "transcrib", "-asr", "asr-", "speech-to-text"),
|
||||
_caps(audio_transcription=True),
|
||||
),
|
||||
# Text-to-speech.
|
||||
(
|
||||
("-tts", "text-to-speech", "voiceclone", "voicedesign"),
|
||||
_caps(audio_speech=True),
|
||||
),
|
||||
# Image generation.
|
||||
(
|
||||
("dall-e", "stable-diffusion", "flux", "imagen", "image-gen"),
|
||||
_caps(images=True),
|
||||
),
|
||||
# Video generation.
|
||||
(("veo-", "sora", "video-gen", "-video"), _caps(video=True)),
|
||||
# Document OCR models — image input, but not a chat endpoint.
|
||||
(("mistral-ocr",), _caps(vision=True)),
|
||||
# Vision-capable chat models (multimodal input).
|
||||
(
|
||||
(
|
||||
"gpt-4o",
|
||||
"gpt-4.1",
|
||||
"gpt-5",
|
||||
"o3",
|
||||
"o4",
|
||||
"claude-3",
|
||||
"claude-4",
|
||||
"gemini",
|
||||
"qwen-vl",
|
||||
"-vl-",
|
||||
"vl-",
|
||||
"vision",
|
||||
"llava",
|
||||
"pixtral",
|
||||
"internvl",
|
||||
"minicpm-v",
|
||||
"llama-3.2-vision",
|
||||
"mimo-vl",
|
||||
# Mistral vision families (BUG-044): text-only mistral-large,
|
||||
# codestral and voxtral are deliberately absent.
|
||||
"ministral",
|
||||
"magistral",
|
||||
"mistral-small",
|
||||
"mistral-medium",
|
||||
"mistral-vibe-cli",
|
||||
"labs-leanstral",
|
||||
),
|
||||
_caps(chat=True, vision=True),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _curated_capabilities(provider: str, model: str) -> dict[str, bool]:
|
||||
"""Curated table lookup (model rules first, then the provider default)."""
|
||||
if model:
|
||||
for needles, caps in _MODEL_RULES:
|
||||
if any(needle in model for needle in needles):
|
||||
return dict(caps)
|
||||
return dict(_PROVIDER_DEFAULTS.get(provider, _caps(chat=True)))
|
||||
|
||||
|
||||
def get_model_capabilities(provider: str, model: str) -> dict[str, bool]:
|
||||
"""Return the capability flags for a ``provider``/``model`` pair.
|
||||
|
||||
A provider declaration (see :mod:`backend.provider_capabilities`) overrides
|
||||
the curated table for every flag it mentions; the curated table supplies the
|
||||
rest.
|
||||
|
||||
Args:
|
||||
provider: Provider identifier (e.g. ``"deepseek"``). Case-insensitive.
|
||||
model: Model identifier (e.g. ``"deepseek-chat"``). May be empty, in
|
||||
which case the provider default is returned.
|
||||
|
||||
Returns:
|
||||
A dict with every key of :data:`CAPABILITY_KEYS` and boolean values.
|
||||
"""
|
||||
provider = (provider or "").strip().lower()
|
||||
name = (model or "").strip().lower()
|
||||
caps = _curated_capabilities(provider, name)
|
||||
declared = get_declared_capabilities(provider, name)
|
||||
if declared:
|
||||
caps.update({key: value for key, value in declared.items() if key in CAPABILITY_KEYS})
|
||||
return caps
|
||||
|
||||
|
||||
def get_capabilities_for_models(
|
||||
provider: str, models: list[str]
|
||||
) -> dict[str, dict[str, bool]]:
|
||||
"""Return a ``{model: capabilities}`` map for a list of models."""
|
||||
return {model: get_model_capabilities(provider, model) for model in models}
|
||||
|
||||
|
||||
def model_supports_vision(provider: str, model: str) -> bool:
|
||||
"""True when the given model can accept image input."""
|
||||
return bool(get_model_capabilities(provider, model).get("vision"))
|
||||
|
||||
|
||||
def capabilities_payload(provider: str, model: str) -> dict[str, Any]:
|
||||
"""Serialize capabilities for API responses."""
|
||||
return {
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"capabilities": get_model_capabilities(provider, model),
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
"""External notifications — Discord, Telegram, SMTP, generic webhook (#168).
|
||||
|
||||
Configuration is persisted in ``data/notify_channels.json``; secrets live in
|
||||
``data/notify_secrets.json`` (0600) or in ``OBSIGATE_NOTIFY_SECRET_<ID>``
|
||||
environment variables — never in the public config file (same pattern as
|
||||
``backend/webhooks.py``, BUG-026).
|
||||
|
||||
Supported channel types:
|
||||
|
||||
* ``discord`` — Discord webhook URL (``POST {"content": ...}``).
|
||||
* ``telegram`` — Bot API (``POST https://api.telegram.org/bot<token>/sendMessage``).
|
||||
* ``smtp`` — Email via stdlib ``smtplib`` (STARTTLS, auth login).
|
||||
* ``webhook`` — generic JSON ``POST`` (SSRF-safe, same policy as #9).
|
||||
|
||||
Each channel declares ``triggers`` chosen among :data:`VALID_TRIGGERS`.
|
||||
The scheduler (#170) broadcasts on ``schedule_failure``; file-event fan-out
|
||||
stays on the historical ``backend/webhooks.py`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import smtplib
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from email.message import EmailMessage
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.notify")
|
||||
|
||||
DATA_DIR = Path(os.environ.get("OBSIGATE_DATA_DIR", "data"))
|
||||
CHANNELS_FILE = DATA_DIR / "notify_channels.json"
|
||||
SECRETS_FILE = DATA_DIR / "notify_secrets.json"
|
||||
|
||||
CHANNEL_TYPES = ("discord", "telegram", "smtp", "webhook")
|
||||
VALID_TRIGGERS = ("manual", "schedule_failure", "schedule_success", "duplicate_found")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
|
||||
# ── Store helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_channels() -> list[dict[str, Any]]:
|
||||
if not CHANNELS_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(CHANNELS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_channels(channels: list[dict[str, Any]]) -> None:
|
||||
CHANNELS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CHANNELS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(channels, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(CHANNELS_FILE)
|
||||
|
||||
|
||||
def _read_secrets() -> dict[str, str]:
|
||||
if not SECRETS_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(SECRETS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_secrets(secrets: dict[str, str]) -> None:
|
||||
SECRETS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = SECRETS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(secrets, indent=2), encoding="utf-8")
|
||||
tmp.replace(SECRETS_FILE)
|
||||
try:
|
||||
SECRETS_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
pass # Windows: pas de permissions Unix
|
||||
|
||||
|
||||
def _secret_key(channel_id: str) -> str:
|
||||
return "OBSIGATE_NOTIFY_SECRET_" + channel_id.replace("-", "_").upper()
|
||||
|
||||
|
||||
def _get_secret(channel_id: str) -> str | None:
|
||||
"""Resolve a channel secret: env > dedicated store > legacy inline config."""
|
||||
env_val = os.environ.get(_secret_key(channel_id))
|
||||
if env_val:
|
||||
return env_val
|
||||
stored = _read_secrets().get(channel_id)
|
||||
if stored:
|
||||
return stored
|
||||
for ch in _read_channels():
|
||||
if ch.get("id") == channel_id:
|
||||
cfg = ch.get("config", {})
|
||||
for key in ("webhook_url", "bot_token", "password"):
|
||||
if cfg.get(key):
|
||||
return str(cfg[key])
|
||||
return None
|
||||
|
||||
|
||||
def _public_view(channel: dict[str, Any]) -> dict[str, Any]:
|
||||
clean = {k: v for k, v in channel.items() if k != "config"}
|
||||
cfg = dict(channel.get("config", {}))
|
||||
for secret_field in ("webhook_url", "bot_token", "password"):
|
||||
if cfg.get(secret_field):
|
||||
cfg[secret_field] = "***"
|
||||
clean["config"] = cfg
|
||||
clean["has_secret"] = bool(_get_secret(channel["id"]))
|
||||
return clean
|
||||
|
||||
|
||||
# ── CRUD ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _validate_config(channel_type: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate (sans secret) and normalize a channel config. Raises ValueError."""
|
||||
config = dict(config or {})
|
||||
if channel_type == "discord":
|
||||
url = str(config.get("webhook_url") or config.get("url") or "").strip()
|
||||
if not url.startswith(("https://discord.com/api/webhooks/", "https://discordapp.com/api/webhooks/")):
|
||||
# Laisse passer les URLs de test locales quand le mode privé est ouvert.
|
||||
from backend.webhooks import validate_webhook_url
|
||||
|
||||
validate_webhook_url(url)
|
||||
if "discord" not in url and not os.environ.get("OBSIGATE_WEBHOOK_ALLOW_PRIVATE"):
|
||||
raise ValueError("URL Discord invalide (webhook discord.com attendu)")
|
||||
config["webhook_url"] = url
|
||||
elif channel_type == "telegram":
|
||||
if not str(config.get("chat_id") or "").strip():
|
||||
raise ValueError("chat_id Telegram requis")
|
||||
config["chat_id"] = str(config["chat_id"]).strip()
|
||||
if config.get("bot_token"):
|
||||
config["bot_token"] = str(config["bot_token"]).strip()
|
||||
elif channel_type == "smtp":
|
||||
for field in ("host", "from_addr", "to_addr"):
|
||||
if not str(config.get(field) or "").strip():
|
||||
raise ValueError(f"Champ SMTP requis : {field}")
|
||||
config["port"] = int(config.get("port") or 587)
|
||||
config["use_tls"] = bool(config.get("use_tls", True))
|
||||
config["username"] = str(config.get("username") or "").strip()
|
||||
elif channel_type == "webhook":
|
||||
from backend.webhooks import validate_webhook_url
|
||||
|
||||
url = str(config.get("url") or "").strip()
|
||||
validate_webhook_url(url)
|
||||
config["url"] = url
|
||||
else:
|
||||
raise ValueError(f"Type de canal inconnu : {channel_type}")
|
||||
triggers = [t for t in (config.get("triggers") or ["manual"]) if t in VALID_TRIGGERS]
|
||||
config["triggers"] = triggers or ["manual"]
|
||||
return config
|
||||
|
||||
|
||||
def list_channels() -> list[dict[str, Any]]:
|
||||
"""Return public views of all notification channels."""
|
||||
return [_public_view(ch) for ch in _read_channels()]
|
||||
|
||||
|
||||
def create_channel(name: str, channel_type: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a notification channel. Secrets are split into the secret store."""
|
||||
if channel_type not in CHANNEL_TYPES:
|
||||
raise ValueError(f"Type de canal inconnu : {channel_type}")
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
channel_id = str(uuid.uuid4())
|
||||
normalized = _validate_config(channel_type, config)
|
||||
secrets = _read_secrets()
|
||||
for field in ("webhook_url", "bot_token", "password"):
|
||||
if normalized.get(field) and len(str(normalized[field])) > 8:
|
||||
secrets[channel_id] = str(normalized[field])
|
||||
normalized[field] = "***" # placeholder : le secret vit dans le store dédié
|
||||
_write_secrets(secrets)
|
||||
channel = {
|
||||
"id": channel_id,
|
||||
"name": (name or channel_type).strip() or channel_type,
|
||||
"type": channel_type,
|
||||
"enabled": True,
|
||||
"config": normalized,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"last_sent_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
channels.append(channel)
|
||||
_write_channels(channels)
|
||||
logger.info(f"Created notify channel '{name}' ({channel_type})")
|
||||
return _public_view(channel)
|
||||
|
||||
|
||||
def update_channel(channel_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a channel (name/enabled/config). Returns None when unknown."""
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
for channel in channels:
|
||||
if channel.get("id") != channel_id:
|
||||
continue
|
||||
if updates.get("name"):
|
||||
channel["name"] = str(updates["name"])
|
||||
if "enabled" in updates:
|
||||
channel["enabled"] = bool(updates["enabled"])
|
||||
if "config" in updates and isinstance(updates["config"], dict):
|
||||
merged = {**channel.get("config", {}), **updates["config"]}
|
||||
normalized = _validate_config(channel["type"], merged)
|
||||
secrets = _read_secrets()
|
||||
for field in ("webhook_url", "bot_token", "password"):
|
||||
if updates["config"].get(field):
|
||||
secrets[channel_id] = str(updates["config"][field])
|
||||
normalized[field] = "***"
|
||||
_write_secrets(secrets)
|
||||
channel["config"] = normalized
|
||||
_write_channels(channels)
|
||||
return _public_view(channel)
|
||||
return None
|
||||
|
||||
|
||||
def delete_channel(channel_id: str) -> bool:
|
||||
"""Delete a channel and its secret. Returns False when unknown."""
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
remaining = [c for c in channels if c.get("id") != channel_id]
|
||||
if len(remaining) == len(channels):
|
||||
return False
|
||||
_write_channels(remaining)
|
||||
secrets = _read_secrets()
|
||||
if secrets.pop(channel_id, None) is not None:
|
||||
_write_secrets(secrets)
|
||||
return True
|
||||
|
||||
|
||||
# ── Dispatch ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _send_discord(webhook_url: str, title: str, message: str) -> None:
|
||||
import httpx
|
||||
|
||||
content = f"**{title}**\n{message}"[:2000]
|
||||
resp = httpx.post(webhook_url, json={"content": content}, timeout=10.0)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def _send_telegram(bot_token: str, chat_id: str, title: str, message: str) -> None:
|
||||
import httpx
|
||||
|
||||
from backend.webhooks import is_safe_target
|
||||
|
||||
url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
|
||||
if not is_safe_target(url):
|
||||
raise RuntimeError("Cible Telegram bloquée par la politique SSRF")
|
||||
text = f"*{title}*\n{message}"[:4000]
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json={"chat_id": chat_id, "text": text, "parse_mode": "Markdown"},
|
||||
timeout=10.0,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def _send_smtp(config: dict[str, Any], password: str | None, title: str, message: str) -> None:
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = f"[ObsiGate] {title}"
|
||||
msg["From"] = config["from_addr"]
|
||||
msg["To"] = config["to_addr"]
|
||||
msg.set_content(message)
|
||||
with smtplib.SMTP(str(config["host"]), int(config.get("port", 587)), timeout=10) as client:
|
||||
if config.get("use_tls", True):
|
||||
client.starttls()
|
||||
if config.get("username") and password:
|
||||
client.login(str(config["username"]), password)
|
||||
client.send_message(msg)
|
||||
|
||||
|
||||
def _send_webhook(url: str, title: str, message: str, trigger: str) -> None:
|
||||
import httpx
|
||||
|
||||
from backend.webhooks import is_safe_target
|
||||
|
||||
if not is_safe_target(url):
|
||||
raise RuntimeError("Cible webhook bloquée par la politique SSRF")
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json={
|
||||
"event": trigger,
|
||||
"title": title,
|
||||
"message": message,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"source": "obsigate-notify",
|
||||
},
|
||||
timeout=10.0,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def send_via_channel(channel: dict[str, Any], title: str, message: str, trigger: str = "manual") -> None:
|
||||
"""Send a notification through one raw channel record. Raises on failure."""
|
||||
channel_type = channel.get("type")
|
||||
cfg = dict(channel.get("config", {}))
|
||||
secret = _get_secret(channel["id"])
|
||||
if channel_type == "discord":
|
||||
url = secret or cfg.get("webhook_url") or ""
|
||||
if not url or url == "***":
|
||||
raise RuntimeError("URL webhook Discord manquante")
|
||||
_send_discord(url, title, message)
|
||||
elif channel_type == "telegram":
|
||||
token = secret or cfg.get("bot_token") or os.environ.get("OBSIGATE_TELEGRAM_BOT_TOKEN") or ""
|
||||
if not token or token == "***":
|
||||
raise RuntimeError("Token bot Telegram manquant")
|
||||
_send_telegram(token, str(cfg.get("chat_id", "")), title, message)
|
||||
elif channel_type == "smtp":
|
||||
_send_smtp(cfg, secret, title, message)
|
||||
elif channel_type == "webhook":
|
||||
url = str(cfg.get("url") or "").strip()
|
||||
if not url:
|
||||
raise RuntimeError("URL webhook manquante")
|
||||
_send_webhook(url, title, message, trigger)
|
||||
else:
|
||||
raise RuntimeError(f"Type de canal inconnu : {channel_type}")
|
||||
|
||||
|
||||
def broadcast(trigger: str, title: str, message: str) -> list[dict[str, Any]]:
|
||||
"""Send to every enabled channel subscribed to *trigger*. Never raises."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for channel in _read_channels():
|
||||
if not channel.get("enabled", True):
|
||||
continue
|
||||
if trigger not in channel.get("config", {}).get("triggers", ["manual"]):
|
||||
continue
|
||||
try:
|
||||
send_via_channel(channel, title, message, trigger)
|
||||
results.append({"channel_id": channel["id"], "ok": True})
|
||||
_mark_sent(channel["id"], None)
|
||||
except Exception as e:
|
||||
logger.warning(f"Notify channel '{channel.get('name')}' failed: {e}")
|
||||
results.append({"channel_id": channel["id"], "ok": False, "error": str(e)})
|
||||
_mark_sent(channel["id"], str(e))
|
||||
return results
|
||||
|
||||
|
||||
def _mark_sent(channel_id: str, error: str | None) -> None:
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
for channel in channels:
|
||||
if channel.get("id") == channel_id:
|
||||
channel["last_sent_at"] = datetime.now(UTC).isoformat()
|
||||
channel["last_error"] = error
|
||||
_write_channels(channels)
|
||||
@@ -0,0 +1,577 @@
|
||||
"""OpenAPI 3.1 documentation helpers for ObsiGate (#72).
|
||||
|
||||
This module keeps the API-documentation concerns out of ``backend/main.py``:
|
||||
|
||||
* :data:`TAGS_METADATA` — human descriptions and ordering for every tag.
|
||||
* :func:`enrich_openapi_schema` — post-processes the schema generated by
|
||||
FastAPI to assign tags, add servers/security/examples and document common
|
||||
error responses.
|
||||
* :func:`render_api_landing` — a self-contained ``/api`` landing page that
|
||||
reads ``/openapi.json`` at runtime and groups endpoints by tag, with direct
|
||||
links to the Swagger UI (``/docs``) and ReDoc (``/redoc``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag metadata
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TAGS_METADATA: list[dict[str, str]] = [
|
||||
{"name": "System", "description": "Health checks, runtime configuration, diagnostics, dashboard stats and the SSE event stream."},
|
||||
{"name": "Auth", "description": "Authentication, sessions, user profile, MFA (TOTP/WebAuthn) and API keys."},
|
||||
{"name": "Files", "description": "Browse, read, create, rename, move, delete and download vault files."},
|
||||
{"name": "PDF", "description": "PDF metadata and byte-range streaming for inline viewing."},
|
||||
{"name": "Vaults", "description": "List, add and remove vaults; per-vault display settings; attachment indexing."},
|
||||
{"name": "Search", "description": "Full-text and advanced search, tags, title suggestions and the link graph."},
|
||||
{"name": "Bookmarks", "description": "Recently opened files, bookmarks and saved searches."},
|
||||
{"name": "ExternalRefs", "description": "Cross-application references from a note to Flowdeck, TikTak, Kanbante or ObsiGate objects (Fusion program, ACC-1.3)."},
|
||||
{"name": "SSO", "description": "draco_session single sign-on emitter: authorize entry point and jti revocation list for the other dracodev apps (Fusion program, ACC-3.1)."},
|
||||
{"name": "Backups", "description": "Automatic file backups, diffs, restore, compression and purge."},
|
||||
{"name": "Export", "description": "Export notes or whole vaults to HTML, Markdown bundle or ePub."},
|
||||
{"name": "Guide", "description": "Download the in-app user guide as Markdown or PDF (mirrors the help modal, FR/EN)."},
|
||||
{"name": "AI", "description": "AI-powered editor actions, provider status and model discovery."},
|
||||
{"name": "BooksLM", "description": "Directory-scoped AI chat (NotebookLM-style) over a vault folder."},
|
||||
{"name": "MCP", "description": "Model Context Protocol server (Streamable HTTP) exposing the shared AI tool layer to external clients (Claude Desktop, Cursor…)."},
|
||||
{"name": "Collaboration", "description": "Real-time collaborative editing over WebSocket (`/ws/collab/{vault}/{path}`): Yjs/CRDT updates, awareness (cursors) and debounced server-side persistence."},
|
||||
{"name": "Sharing", "description": "Create and manage public read-only share links for documents."},
|
||||
{"name": "Webhooks", "description": "HTTP callbacks signed with HMAC-SHA256 for file events."},
|
||||
{"name": "Conflicts", "description": "Detect and resolve Syncthing sync-conflict files."},
|
||||
{"name": "Admin", "description": "Admin-only system monitoring: stats, audit log, backup stats and live stream."},
|
||||
{"name": "Coffres Secrets", "description": "Secret vaults: encrypted coffres, secrets, consumers, grants and distribution — metadata only, values are never exposed."},
|
||||
{"name": "Plugins", "description": "Install, enable and manage user plugins."},
|
||||
{"name": "Push", "description": "Web Push (VAPID) subscription management and test notifications."},
|
||||
{"name": "Duplicates", "description": "Duplicate-note detection and confirmed merge (#166)."},
|
||||
{"name": "Notify", "description": "External notifications: Discord, Telegram, SMTP and generic webhooks (#168)."},
|
||||
{"name": "Scheduler", "description": "Scheduled automatic tasks reusing the vault mutation services (#170)."},
|
||||
{"name": "Frontend", "description": "Static assets and SPA fallback routes."},
|
||||
]
|
||||
|
||||
API_DESCRIPTION = """
|
||||
**ObsiGate** exposes a REST API covering the entire application: vaults, files,
|
||||
full-text search, backups, exports, AI actions, sharing and administration.
|
||||
|
||||
### Authentication
|
||||
When authentication is enabled, send a bearer token obtained from
|
||||
`POST /api/auth/login`:
|
||||
|
||||
```
|
||||
Authorization: Bearer <access_token>
|
||||
```
|
||||
|
||||
The same token is also accepted as an HTTP-only cookie, so browser clients can
|
||||
simply use `credentials: "include"`.
|
||||
|
||||
### Real-time collaboration
|
||||
Besides the REST API, a WebSocket endpoint `GET /ws/collab/{vault}/{path}` (upgrade) powers
|
||||
simultaneous editing of the same file: clients exchange Yjs/CRDT updates and awareness (remote
|
||||
cursors), and the server persists the document 2 s after the last change. Authentication uses the
|
||||
`access_token` cookie (or a `token` query parameter) and vault access is enforced per connection.
|
||||
See the collaboration feature documentation (`docs/features/collaboration.md`) for the protocol.
|
||||
|
||||
### Interactive documentation
|
||||
* **Swagger UI** — [/docs](/docs): try requests directly from the browser.
|
||||
* **ReDoc** — [/redoc](/redoc): clean, reading-oriented reference.
|
||||
* **OpenAPI JSON** — [/openapi.json](/openapi.json): machine-readable schema.
|
||||
|
||||
### Errors
|
||||
Errors use the standard FastAPI envelope `{"detail": "..."}` with the
|
||||
appropriate HTTP status (`400`, `401`, `403`, `404`, `409`, `422`, `500`).
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Automatic tag assignment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Rules are evaluated in order; the first match wins. Keep the most specific
|
||||
# prefixes first (e.g. BooksLM before AI, config/ai-* before config).
|
||||
_TAG_RULES: list[tuple[re.Pattern[str], str]] = [
|
||||
(re.compile(r"^/api/auth"), "Auth"),
|
||||
(re.compile(r"^/api/admin"), "Admin"),
|
||||
(re.compile(r"^/api/plugins"), "Plugins"),
|
||||
(re.compile(r"^/api/push"), "Push"),
|
||||
(re.compile(r"^/api/ai/bookslm"), "BooksLM"),
|
||||
(re.compile(r"^/api/ai"), "AI"),
|
||||
(re.compile(r"^/mcp"), "MCP"),
|
||||
(re.compile(r"^/api/config/ai-"), "AI"),
|
||||
(re.compile(r"^/api/share"), "Sharing"),
|
||||
(re.compile(r"^/api/shares"), "Sharing"),
|
||||
(re.compile(r"^/s/"), "Sharing"),
|
||||
(re.compile(r"^/api/webhooks"), "Webhooks"),
|
||||
(re.compile(r"^/api/duplicates"), "Duplicates"),
|
||||
(re.compile(r"^/api/notify"), "Notify"),
|
||||
(re.compile(r"^/api/scheduler"), "Scheduler"),
|
||||
(re.compile(r"^/api/conflicts"), "Conflicts"),
|
||||
(re.compile(r"^/api/backups"), "Backups"),
|
||||
(re.compile(r"^/api/file/[^/]+/(backups|diff|restore)"), "Backups"),
|
||||
(re.compile(r"^/api/export"), "Export"),
|
||||
(re.compile(r"^/api/guide"), "Guide"),
|
||||
(re.compile(r"^/api/file/[^/]+/pdf"), "PDF"),
|
||||
(re.compile(r"^/api/search"), "Search"),
|
||||
(re.compile(r"^/api/tags"), "Search"),
|
||||
(re.compile(r"^/api/tree-search"), "Search"),
|
||||
(re.compile(r"^/api/suggest"), "Search"),
|
||||
(re.compile(r"^/api/graph"), "Search"),
|
||||
(re.compile(r"^/api/recent"), "Bookmarks"),
|
||||
(re.compile(r"^/api/bookmarks"), "Bookmarks"),
|
||||
(re.compile(r"^/api/refs"), "ExternalRefs"),
|
||||
(re.compile(r"^/api/sso"), "SSO"),
|
||||
(re.compile(r"^/api/saved-searches"), "Bookmarks"),
|
||||
(re.compile(r"^/api/vaults"), "Vaults"),
|
||||
(re.compile(r"^/api/vault/"), "Vaults"),
|
||||
(re.compile(r"^/api/index/reload"), "Vaults"),
|
||||
(re.compile(r"^/api/attachments"), "Vaults"),
|
||||
(re.compile(r"^/api/browse"), "Files"),
|
||||
(re.compile(r"^/api/file"), "Files"),
|
||||
(re.compile(r"^/api/directory"), "Files"),
|
||||
(re.compile(r"^/api/move"), "Files"),
|
||||
(re.compile(r"^/api/image"), "Files"),
|
||||
(re.compile(r"^/api/health"), "System"),
|
||||
(re.compile(r"^/api/config"), "System"),
|
||||
(re.compile(r"^/api/diagnostics"), "System"),
|
||||
(re.compile(r"^/api/dashboard"), "System"),
|
||||
(re.compile(r"^/api/events"), "System"),
|
||||
]
|
||||
|
||||
|
||||
def tag_for_path(path: str) -> str:
|
||||
"""Return the documentation tag for an API path (``Frontend`` as fallback)."""
|
||||
for pattern, tag in _TAG_RULES:
|
||||
if pattern.match(path):
|
||||
return tag
|
||||
return "Frontend"
|
||||
|
||||
|
||||
# Normalise tags declared by individual routers (``tags=["auth"]``) to the
|
||||
# canonical, documented tag names.
|
||||
_TAG_ALIASES: dict[str, str] = {
|
||||
"auth": "Auth",
|
||||
"admin": "Admin",
|
||||
"plugins": "Plugins",
|
||||
"push": "Push",
|
||||
"files": "Files",
|
||||
"vaults": "Vaults",
|
||||
"search": "Search",
|
||||
"backups": "Backups",
|
||||
"export": "Export",
|
||||
"sharing": "Sharing",
|
||||
"webhooks": "Webhooks",
|
||||
"duplicates": "Duplicates",
|
||||
"notify": "Notify",
|
||||
"scheduler": "Scheduler",
|
||||
"conflicts": "Conflicts",
|
||||
"system": "System",
|
||||
"frontend": "Frontend",
|
||||
"ai": "AI",
|
||||
"bookslm": "BooksLM",
|
||||
"mcp": "MCP",
|
||||
"pdf": "PDF",
|
||||
"bookmarks": "Bookmarks",
|
||||
}
|
||||
|
||||
_CANONICAL_TAGS = {tag["name"] for tag in TAGS_METADATA}
|
||||
|
||||
|
||||
def canonical_tag(name: str) -> str:
|
||||
"""Map a router-declared tag to its documented name (unchanged if unknown)."""
|
||||
mapped = _TAG_ALIASES.get(name.lower(), name)
|
||||
return mapped if mapped in _CANONICAL_TAGS else name
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request/response examples injected into the schema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ENDPOINT_EXAMPLES: dict[tuple[str, str], dict[str, Any]] = {
|
||||
("post", "/api/file/{vault_name}"): {
|
||||
"request": {"path": "notes/Nouvelle note.md", "content": "# Titre\n\nContenu"},
|
||||
"response": {"success": True, "path": "notes/Nouvelle note.md"},
|
||||
},
|
||||
("put", "/api/file/{vault_name}/save"): {
|
||||
"request": {"path": "notes/Accueil.md", "content": "# Accueil\n\nMis à jour."},
|
||||
"response": {"status": "ok", "vault": "TestVault", "path": "notes/Accueil.md", "size": 26},
|
||||
},
|
||||
("put", "/api/file/{vault_name}/xlsx/save"): {
|
||||
"request": {"sheet": "Budget", "cells": {"B1": "250"}, "allow_formula": False, "force": False},
|
||||
"response": {"status": "ok", "vault": "TestVault", "path": "data/budget.xlsx", "size": 1},
|
||||
},
|
||||
("put", "/api/file/{vault_name}/xlsx/style"): {
|
||||
"request": {
|
||||
"ops": [
|
||||
{"op": "cell", "sheet": "Budget", "range": "A1:B1", "style": {"bold": True, "fill_color": "#ffe08a"}},
|
||||
{"op": "col_width", "sheet": "Budget", "col": "A", "width": 24},
|
||||
],
|
||||
"force": False,
|
||||
"if_match": "18f2c0ab-1f4",
|
||||
},
|
||||
"response": {"status": "ok", "vault": "TestVault", "path": "data/budget.xlsx", "size": 2, "revision": "18f2c0ab-1f6"},
|
||||
},
|
||||
# GET : pas d'exemple de requête (un requestBody sur un GET serait un OpenAPI
|
||||
# invalide) — les paramètres sont documentés par leurs Query().
|
||||
("get", "/api/file/{vault_name}/xlsx/sheet"): {
|
||||
"response": {
|
||||
"vault": "TestVault",
|
||||
"path": "data/budget.xlsx",
|
||||
"sheet": "Budget",
|
||||
"offset": 0,
|
||||
"limit": 200,
|
||||
"rows": 2,
|
||||
"cols": 2,
|
||||
"total_rows": 640,
|
||||
"total_cols": 12,
|
||||
"max_rows": 500,
|
||||
"max_cols": 40,
|
||||
"truncated": True,
|
||||
"has_more": True,
|
||||
"html": "<table>…</table>",
|
||||
},
|
||||
},
|
||||
("post", "/api/search/replace"): {
|
||||
"request": {"query": "Python", "replacement": "Python 3", "vault": "all", "dry_run": True},
|
||||
"response": {"matches": [{"vault": "TestVault", "path": "note1.md", "title": "Python", "match_count": 3}], "total_matches": 3, "dry_run": True},
|
||||
},
|
||||
("post", "/api/ai/improve"): {
|
||||
"request": {"text": "ce texte est bof", "provider": "deepseek"},
|
||||
"response": {"result": "Ce texte mérite d'être amélioré.", "provider": "deepseek"},
|
||||
},
|
||||
("post", "/api/ai/bookslm/chat"): {
|
||||
"request": {"vault": "TestVault", "directory": "projets", "message": "Résume ce dossier", "conversation_history": []},
|
||||
"response": {"token": "Voici un résumé…", "provider": "deepseek", "model": "deepseek-chat"},
|
||||
},
|
||||
("post", "/api/share/{vault_name}"): {
|
||||
"request": {"path": "notes/Accueil.md", "expires_in_hours": 168},
|
||||
"response": {"id": "abc…", "token": "abc…", "vault": "TestVault", "path": "notes/Accueil.md", "url": "/s/abc…"},
|
||||
},
|
||||
("post", "/api/webhooks"): {
|
||||
"request": {"name": "CI", "url": "https://example.com/hook", "events": ["file_modified"], "secret": "s3cr3t", "vault": "TikTak", "path_prefix": "Inbox"},
|
||||
"response": {"id": "3f2c…", "name": "CI", "url": "https://example.com/hook", "events": ["file_modified"], "enabled": True, "vault": "TikTak", "path_prefix": "Inbox"},
|
||||
},
|
||||
}
|
||||
|
||||
# Common error responses documented on every operation.
|
||||
_COMMON_ERRORS: dict[str, dict[str, Any]] = {
|
||||
"401": {
|
||||
"description": "Authentication required or token expired",
|
||||
"content": {"application/json": {"example": {"detail": "Not authenticated"}}},
|
||||
},
|
||||
"403": {
|
||||
"description": "Access to the vault is denied",
|
||||
"content": {"application/json": {"example": {"detail": "Accès refusé à la vault 'Notes'"}}},
|
||||
},
|
||||
"404": {
|
||||
"description": "Resource not found",
|
||||
"content": {"application/json": {"example": {"detail": "File not found: notes/x.md"}}},
|
||||
},
|
||||
"422": {
|
||||
"description": "Validation error",
|
||||
"content": {"application/json": {"example": {"detail": [{"loc": ["query", "path"], "msg": "field required", "type": "value_error.missing"}]}}},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"content": {"application/json": {"example": {"detail": "Internal server error"}}},
|
||||
},
|
||||
}
|
||||
|
||||
# Paths that return a binary/streaming body — never add a JSON error example
|
||||
# to their 200 response, and skip the 500 JSON example for streams.
|
||||
_BINARY_MEDIA = ("application/pdf", "application/octet-stream", "image/", "text/event-stream")
|
||||
|
||||
|
||||
def _is_binary_operation(operation: dict[str, Any]) -> bool:
|
||||
content = operation.get("responses", {}).get("200", {}).get("content", {})
|
||||
return any(media.startswith(_BINARY_MEDIA) for media in content)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCP endpoint (not a FastAPI route: custom ASGI mount) — documented manually
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MCP_DESCRIPTION = (
|
||||
"**Model Context Protocol** server over Streamable HTTP (JSON-RPC 2.0). "
|
||||
"Exposes the shared AI tool layer to external MCP clients (Claude Desktop, "
|
||||
"Cursor…). Authentication uses `Authorization: Bearer <JWT>` (the same "
|
||||
"token as the REST API).\n\n"
|
||||
"Primitives: read/search **tools** directly; write/destructive tools as a "
|
||||
"two-step `propose_<tool>` / `apply_<tool>` pair (signed, single-use "
|
||||
"confirmation token); **resources** `vault://<name>` and "
|
||||
"`vault://<name>/<path>` (read-only, secrets redacted); **prompts** "
|
||||
"`summarize-directory`, `generate-note`, `find-related`.\n\n"
|
||||
"See `docs/MCP_GUIDE.md` for client setup."
|
||||
)
|
||||
|
||||
|
||||
def _inject_mcp_path(schema: dict[str, Any]) -> None:
|
||||
"""Add the MCP Streamable HTTP endpoint to the schema (idempotent)."""
|
||||
paths = schema.setdefault("paths", {})
|
||||
if "/mcp" in paths:
|
||||
return
|
||||
paths["/mcp"] = {
|
||||
"post": {
|
||||
"tags": ["MCP"],
|
||||
"summary": "MCP Streamable HTTP endpoint (JSON-RPC 2.0)",
|
||||
"operationId": "mcp_streamable_http",
|
||||
"description": _MCP_DESCRIPTION,
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "tools/list",
|
||||
"params": {},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "JSON-RPC response (or 202 for notifications)",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"result": {"tools": []},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
"security": [{"bearerAuth": []}],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def enrich_openapi_schema(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Enrich a FastAPI-generated OpenAPI schema in place and return it.
|
||||
|
||||
The transformation is idempotent: calling it twice yields the same result.
|
||||
"""
|
||||
schema.setdefault("openapi", "3.1.0")
|
||||
info = schema.setdefault("info", {})
|
||||
info.setdefault("description", API_DESCRIPTION.strip())
|
||||
info.setdefault("contact", {"name": "ObsiGate", "url": "https://git.dracodev.net/Projets/ObsiGate"})
|
||||
info.setdefault("license", {"name": "MIT"})
|
||||
|
||||
schema.setdefault("servers", [{"url": "/", "description": "Current ObsiGate instance"}])
|
||||
schema["externalDocs"] = {
|
||||
"description": "ObsiGate source repository",
|
||||
"url": "https://git.dracodev.net/Projets/ObsiGate",
|
||||
}
|
||||
schema["tags"] = TAGS_METADATA
|
||||
|
||||
_inject_mcp_path(schema)
|
||||
|
||||
components = schema.setdefault("components", {})
|
||||
security_schemes = components.setdefault("securitySchemes", {})
|
||||
security_schemes.setdefault("bearerAuth", {
|
||||
"type": "http",
|
||||
"scheme": "bearer",
|
||||
"bearerFormat": "JWT",
|
||||
"description": "JWT access token issued by POST /api/auth/login.",
|
||||
})
|
||||
security_schemes.setdefault("cookieAuth", {
|
||||
"type": "apiKey",
|
||||
"in": "cookie",
|
||||
"name": "obsigate_token",
|
||||
"description": "HTTP-only session cookie (browser clients).",
|
||||
})
|
||||
components.setdefault("schemas", {}).setdefault("ErrorResponse", {
|
||||
"type": "object",
|
||||
"properties": {"detail": {"title": "Detail", "description": "Error message"}},
|
||||
})
|
||||
|
||||
for path, operations in schema.get("paths", {}).items():
|
||||
default_tag = tag_for_path(path)
|
||||
for method, operation in operations.items():
|
||||
if not isinstance(operation, dict):
|
||||
continue
|
||||
if operation.get("tags"):
|
||||
operation["tags"] = [canonical_tag(t) for t in operation["tags"]]
|
||||
else:
|
||||
operation["tags"] = [default_tag]
|
||||
|
||||
# Document common error responses without overriding explicit ones.
|
||||
responses = operation.setdefault("responses", {})
|
||||
binary = _is_binary_operation(operation)
|
||||
for code, spec in _COMMON_ERRORS.items():
|
||||
if code == "500" and binary:
|
||||
continue
|
||||
responses.setdefault(code, spec)
|
||||
|
||||
# Inject request/response examples for key endpoints.
|
||||
example = _ENDPOINT_EXAMPLES.get((method, path))
|
||||
if example:
|
||||
if "request" in example:
|
||||
body = operation.setdefault("requestBody", {})
|
||||
content = body.setdefault("content", {}).setdefault("application/json", {})
|
||||
content.setdefault("example", example["request"])
|
||||
if "response" in example:
|
||||
ok = responses.setdefault("200", {})
|
||||
content = ok.setdefault("content", {}).setdefault("application/json", {})
|
||||
content.setdefault("example", example["response"])
|
||||
|
||||
return schema
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api landing page
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_METHOD_COLORS = {
|
||||
"get": "#2563eb",
|
||||
"post": "#16a34a",
|
||||
"put": "#d97706",
|
||||
"patch": "#7c3aed",
|
||||
"delete": "#dc2626",
|
||||
"head": "#64748b",
|
||||
"options": "#64748b",
|
||||
}
|
||||
|
||||
|
||||
def render_api_landing(version: str = "") -> str:
|
||||
"""Return the self-contained HTML for the ``/api`` documentation page."""
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>ObsiGate API{(' — v' + version) if version else ''}</title>
|
||||
<style>
|
||||
:root {{
|
||||
--bg: #0f172a; --surface: #1e293b; --border: #334155;
|
||||
--text: #e2e8f0; --muted: #94a3b8; --accent: #60a5fa; --accent-2: #34d399;
|
||||
}}
|
||||
html[data-theme="light"] {{
|
||||
--bg: #f8fafc; --surface: #ffffff; --border: #e2e8f0;
|
||||
--text: #0f172a; --muted: #64748b; --accent: #2563eb; --accent-2: #059669;
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
margin: 0; background: var(--bg); color: var(--text);
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.5;
|
||||
}}
|
||||
.wrap {{ max-width: 1080px; margin: 0 auto; padding: 40px 24px 80px; }}
|
||||
header h1 {{ margin: 0 0 4px; font-size: 1.9rem; }}
|
||||
header p {{ margin: 0; color: var(--muted); }}
|
||||
.actions {{ display: flex; flex-wrap: wrap; gap: 10px; margin: 24px 0 32px; }}
|
||||
.actions a {{
|
||||
text-decoration: none; color: var(--text); background: var(--surface);
|
||||
border: 1px solid var(--border); border-radius: 8px; padding: 10px 16px;
|
||||
font-weight: 600; font-size: .9rem; transition: border-color .15s, transform .15s;
|
||||
}}
|
||||
.actions a:hover {{ border-color: var(--accent); transform: translateY(-1px); }}
|
||||
.actions a.primary {{ background: var(--accent); border-color: var(--accent); color: #fff; }}
|
||||
.meta {{ color: var(--muted); font-size: .85rem; margin-bottom: 24px; }}
|
||||
.tag {{ margin: 28px 0; }}
|
||||
.tag > h2 {{
|
||||
font-size: 1.1rem; margin: 0 0 2px; display: flex; align-items: center; gap: 8px;
|
||||
}}
|
||||
.tag > p {{ margin: 0 0 12px; color: var(--muted); font-size: .85rem; }}
|
||||
.ops {{ display: flex; flex-direction: column; gap: 6px; }}
|
||||
.op {{
|
||||
display: flex; align-items: center; gap: 12px; padding: 8px 12px;
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
|
||||
text-decoration: none; color: var(--text); font-size: .88rem;
|
||||
}}
|
||||
.op:hover {{ border-color: var(--accent); }}
|
||||
.method {{
|
||||
flex: 0 0 62px; text-align: center; font-weight: 700; font-size: .68rem;
|
||||
text-transform: uppercase; letter-spacing: .04em; color: #fff; border-radius: 5px; padding: 3px 0;
|
||||
}}
|
||||
.op code {{ color: var(--accent-2); font-size: .85rem; }}
|
||||
.op .summary {{ color: var(--muted); margin-left: auto; text-align: right; }}
|
||||
.loading {{ color: var(--muted); }}
|
||||
footer {{ margin-top: 48px; color: var(--muted); font-size: .8rem; }}
|
||||
@media (max-width: 640px) {{ .op .summary {{ display: none; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header>
|
||||
<h1>ObsiGate API</h1>
|
||||
<p>Interactive reference for the ObsiGate REST API (OpenAPI 3.1).</p>
|
||||
</header>
|
||||
<div class="actions">
|
||||
<a class="primary" href="/docs">Swagger UI — try it</a>
|
||||
<a href="/redoc">ReDoc — reference</a>
|
||||
<a href="/openapi.json">OpenAPI JSON</a>
|
||||
<a href="/">← Back to ObsiGate</a>
|
||||
</div>
|
||||
<div class="meta" id="meta">Loading specification…</div>
|
||||
<div id="tags"><p class="loading">Loading endpoints…</p></div>
|
||||
<footer>Generated from the live OpenAPI schema. Authentication: <code>Authorization: Bearer <token></code>.</footer>
|
||||
</div>
|
||||
<script>
|
||||
(function () {{
|
||||
var COLORS = {_js_colors()};
|
||||
function el(tag, cls, text) {{
|
||||
var e = document.createElement(tag);
|
||||
if (cls) e.className = cls;
|
||||
if (text != null) e.textContent = text;
|
||||
return e;
|
||||
}}
|
||||
fetch('/openapi.json', {{ credentials: 'include' }})
|
||||
.then(function (r) {{ if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); }})
|
||||
.then(function (spec) {{
|
||||
var info = spec.info || {{}};
|
||||
var tagMeta = {{}};
|
||||
(spec.tags || []).forEach(function (t) {{ tagMeta[t.name] = t.description || ''; }});
|
||||
var groups = {{}};
|
||||
var paths = spec.paths || {{}};
|
||||
var opCount = 0;
|
||||
Object.keys(paths).forEach(function (path) {{
|
||||
var ops = paths[path];
|
||||
Object.keys(ops).forEach(function (method) {{
|
||||
if (['get','post','put','patch','delete','head','options'].indexOf(method) === -1) return;
|
||||
var op = ops[method];
|
||||
if (op.deprecated && false) return;
|
||||
opCount++;
|
||||
var tag = (op.tags && op.tags[0]) || 'Other';
|
||||
(groups[tag] = groups[tag] || []).push({{ path: path, method: method, summary: op.summary || '' }});
|
||||
}});
|
||||
}});
|
||||
document.getElementById('meta').textContent =
|
||||
(info.title || 'ObsiGate') + (info.version ? ' v' + info.version : '') +
|
||||
' — OpenAPI ' + (spec.openapi || '3.1') + ' — ' + opCount + ' operations';
|
||||
var host = document.getElementById('tags');
|
||||
host.innerHTML = '';
|
||||
Object.keys(groups).forEach(function (tag) {{
|
||||
var section = el('section', 'tag');
|
||||
section.appendChild(el('h2', null, tag));
|
||||
if (tagMeta[tag]) section.appendChild(el('p', null, tagMeta[tag]));
|
||||
var ops = el('div', 'ops');
|
||||
groups[tag].forEach(function (o) {{
|
||||
var a = el('a', 'op');
|
||||
a.href = '/docs#/' + encodeURIComponent(tag) + '/' + encodeURIComponent(o.path);
|
||||
var m = el('span', 'method', o.method);
|
||||
m.style.background = COLORS[o.method] || '#64748b';
|
||||
a.appendChild(m);
|
||||
a.appendChild(el('code', null, o.path));
|
||||
if (o.summary) a.appendChild(el('span', 'summary', o.summary));
|
||||
ops.appendChild(a);
|
||||
}});
|
||||
section.appendChild(ops);
|
||||
host.appendChild(section);
|
||||
}});
|
||||
}})
|
||||
.catch(function (e) {{
|
||||
document.getElementById('meta').textContent = 'Failed to load OpenAPI schema: ' + e.message;
|
||||
}});
|
||||
}})();
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _js_colors() -> str:
|
||||
"""Serialise the method→colour map for the landing page script."""
|
||||
import json
|
||||
|
||||
return json.dumps(_METHOD_COLORS)
|
||||
@@ -43,7 +43,7 @@ def build_pdf_html(body_html: str, title: str, theme: str = "light") -> str:
|
||||
<head><meta charset="utf-8"><title>{title}</title>
|
||||
<style>
|
||||
body {{
|
||||
font-family: Georgia, "Times New Roman", serif;
|
||||
font-family: Georgia, "Times New Roman", serif, "Noto Color Emoji";
|
||||
max-width: 720px;
|
||||
margin: 40px auto;
|
||||
padding: 0 20px;
|
||||
|
||||
+53
-5
@@ -2,11 +2,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configurable limits (see ROADMAP #74 G3):
|
||||
# OBSIGATE_PDF_MAX_SIZE_MB — PDFs larger than this are not text-extracted (default 50)
|
||||
# OBSIGATE_PDF_EXTRACT_TIMEOUT — seconds before extraction is abandoned (default 30)
|
||||
PDF_MAX_SIZE_MB: int = int(os.environ.get("OBSIGATE_PDF_MAX_SIZE_MB", "50"))
|
||||
PDF_EXTRACT_TIMEOUT: float = float(os.environ.get("OBSIGATE_PDF_EXTRACT_TIMEOUT", "30"))
|
||||
|
||||
PDF_READER: str = "pypdf"
|
||||
PdfReader: Any = None
|
||||
try:
|
||||
import fitz # pymupdf
|
||||
PDF_READER = "pymupdf"
|
||||
@@ -16,19 +27,54 @@ except ImportError:
|
||||
from pypdf import PdfReader # type: ignore
|
||||
logger.info("PDF reader: pypdf (pure Python)")
|
||||
except ImportError:
|
||||
PdfReader = None # type: ignore
|
||||
logger.warning("No PDF reader available — install pypdf or pymupdf")
|
||||
|
||||
|
||||
def pdf_exceeds_size_limit(file_path: Path) -> bool:
|
||||
"""True if the PDF is larger than OBSIGATE_PDF_MAX_SIZE_MB (skip text extraction)."""
|
||||
try:
|
||||
return file_path.stat().st_size > PDF_MAX_SIZE_MB * 1024 * 1024
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _run_with_timeout(fn, *args, timeout: float):
|
||||
"""Run a sync function in a worker thread with a hard timeout.
|
||||
|
||||
A timed-out extraction leaves its worker thread running (daemon-style pool
|
||||
is abandoned), but the request itself is freed — acceptable trade-off for
|
||||
pathological PDFs on a self-hosted single-user server.
|
||||
"""
|
||||
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pdf-extract")
|
||||
try:
|
||||
future = executor.submit(fn, *args)
|
||||
return future.result(timeout=timeout)
|
||||
finally:
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
|
||||
def extract_pdf_text(file_path: Path, max_chars: int = 100000) -> str:
|
||||
"""Extract text from a PDF file. Returns empty string on failure."""
|
||||
"""Extract text from a PDF file. Returns empty string on failure.
|
||||
|
||||
Oversized PDFs (> OBSIGATE_PDF_MAX_SIZE_MB) and extractions exceeding
|
||||
OBSIGATE_PDF_EXTRACT_TIMEOUT seconds return "" instead of blocking.
|
||||
"""
|
||||
if PdfReader is None and PDF_READER == "pypdf":
|
||||
return ""
|
||||
if pdf_exceeds_size_limit(file_path):
|
||||
logger.info("PDF too large to index (> %d MB), skipping text extraction: %s",
|
||||
PDF_MAX_SIZE_MB, file_path)
|
||||
return ""
|
||||
try:
|
||||
if PDF_READER == "pymupdf":
|
||||
return _extract_pymupdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pymupdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
else:
|
||||
return _extract_pypdf(file_path, max_chars)
|
||||
return _run_with_timeout(_extract_pypdf, file_path, max_chars,
|
||||
timeout=PDF_EXTRACT_TIMEOUT)
|
||||
except FuturesTimeout:
|
||||
logger.warning("PDF text extraction timed out (%ss): %s", PDF_EXTRACT_TIMEOUT, file_path)
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.warning("Failed to extract PDF text from %s: %s", file_path, e)
|
||||
return ""
|
||||
@@ -45,7 +91,7 @@ def extract_pdf_metadata(file_path: Path) -> dict:
|
||||
info["title"] = meta.get("title", "")
|
||||
info["author"] = meta.get("author", "")
|
||||
doc.close()
|
||||
else:
|
||||
elif PdfReader is not None:
|
||||
reader = PdfReader(str(file_path))
|
||||
info["pages"] = len(reader.pages)
|
||||
meta = reader.metadata or {}
|
||||
@@ -114,6 +160,8 @@ def _extract_pymupdf(file_path: Path, max_chars: int) -> str:
|
||||
|
||||
|
||||
def _extract_pypdf(file_path: Path, max_chars: int) -> str:
|
||||
if PdfReader is None:
|
||||
return ""
|
||||
reader = PdfReader(str(file_path))
|
||||
parts = []
|
||||
total = 0
|
||||
|
||||
@@ -0,0 +1,624 @@
|
||||
"""Plugin system for ObsiGate — backend API and manifest validation.
|
||||
|
||||
Plugins extend ObsiGate with custom renderers, search filters, and editor
|
||||
actions. They run sandboxed in a Web Worker on the frontend; the backend
|
||||
handles manifest validation, storage, and lifecycle (install / uninstall /
|
||||
enable / disable), all scoped to a single vault.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time as _time
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_admin, require_auth
|
||||
from backend.indexer import get_vault_data
|
||||
|
||||
logger = logging.getLogger("obsigate.plugins")
|
||||
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
|
||||
|
||||
# ── Constants ──────────────────────────────────────────────────────────────
|
||||
PLUGINS_DIR_NAME = ".obsigate-plugins"
|
||||
MANIFEST_FILENAME = "plugin.json"
|
||||
MAX_PLUGIN_SIZE = 500_000 # 500 KB per plugin file
|
||||
MAX_PLUGINS_PER_VAULT = 50
|
||||
MAX_PLUGIN_FILES = 100 # max files in an installed plugin zip
|
||||
|
||||
ALLOWED_HOOKS = frozenset([
|
||||
"onFileRender",
|
||||
"onSearchFilter",
|
||||
"onEditorAction",
|
||||
"onSidebarItem",
|
||||
"onFileCreate",
|
||||
"onFileDelete",
|
||||
"onVaultMount",
|
||||
])
|
||||
|
||||
ALLOWED_PERMISSIONS = frozenset([
|
||||
"read_files",
|
||||
"write_files",
|
||||
"read_vault_metadata",
|
||||
"network_request",
|
||||
"ui_notify",
|
||||
"access_clipboard",
|
||||
])
|
||||
|
||||
|
||||
# ── Manifest ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class PluginManifest:
|
||||
"""Validated plugin manifest from plugin.json."""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
description: str
|
||||
author: str
|
||||
main: str
|
||||
hooks: dict[str, str] = field(default_factory=dict)
|
||||
permissions: list[str] = field(default_factory=list)
|
||||
min_obsigate_version: str = "2.1.0"
|
||||
homepage: str | None = None
|
||||
repository: str | None = None
|
||||
license: str = "MIT"
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "PluginManifest":
|
||||
"""Validate and build a manifest from a raw dict (JSON object)."""
|
||||
required = ["name", "version", "description", "author", "main"]
|
||||
for field_name in required:
|
||||
if field_name not in data or not data[field_name]:
|
||||
raise ValueError(f"Missing required field: {field_name}")
|
||||
|
||||
name = data["name"]
|
||||
if not re.fullmatch(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?", name):
|
||||
raise ValueError(
|
||||
"Plugin name must be lowercase alphanumeric with hyphens (e.g., 'my-plugin')"
|
||||
)
|
||||
|
||||
version = data["version"]
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+(-[a-zA-Z0-9.-]+)?", version):
|
||||
raise ValueError("Version must be semantic version (e.g., '1.0.0')")
|
||||
|
||||
hooks = data.get("hooks", {})
|
||||
if not isinstance(hooks, dict):
|
||||
raise TypeError("'hooks' must be an object mapping hook name to handler name")
|
||||
for hook_name, handler in hooks.items():
|
||||
if hook_name not in ALLOWED_HOOKS:
|
||||
raise ValueError(
|
||||
f"Unknown hook: {hook_name}. Allowed: {sorted(ALLOWED_HOOKS)}"
|
||||
)
|
||||
if not isinstance(handler, str) or not handler:
|
||||
raise ValueError(f"Hook handler for '{hook_name}' must be a non-empty string")
|
||||
|
||||
permissions = data.get("permissions", [])
|
||||
if not isinstance(permissions, list):
|
||||
raise TypeError("'permissions' must be a list of permission names")
|
||||
for perm in permissions:
|
||||
if perm not in ALLOWED_PERMISSIONS:
|
||||
raise ValueError(
|
||||
f"Unknown permission: {perm}. Allowed: {sorted(ALLOWED_PERMISSIONS)}"
|
||||
)
|
||||
|
||||
return cls(
|
||||
name=name,
|
||||
version=version,
|
||||
description=data["description"],
|
||||
author=data["author"],
|
||||
main=data["main"],
|
||||
hooks=hooks,
|
||||
permissions=permissions,
|
||||
min_obsigate_version=data.get("min_obsigate_version", "2.1.0"),
|
||||
homepage=data.get("homepage"),
|
||||
repository=data.get("repository"),
|
||||
license=data.get("license", "MIT"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"version": self.version,
|
||||
"description": self.description,
|
||||
"author": self.author,
|
||||
"main": self.main,
|
||||
"hooks": self.hooks,
|
||||
"permissions": self.permissions,
|
||||
"min_obsigate_version": self.min_obsigate_version,
|
||||
"homepage": self.homepage,
|
||||
"repository": self.repository,
|
||||
"license": self.license,
|
||||
}
|
||||
|
||||
|
||||
# ── Storage path helpers ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _vault_plugins_dir(vault_name: str) -> Path:
|
||||
"""Return the plugins directory for a vault (creating it if absent)."""
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
base = Path(vault_data["path"])
|
||||
pd = base / PLUGINS_DIR_NAME
|
||||
pd.mkdir(parents=True, exist_ok=True)
|
||||
return pd
|
||||
|
||||
|
||||
# ── Manager ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class PluginManager:
|
||||
"""Manages plugin lifecycle within a single plugins directory."""
|
||||
|
||||
def __init__(self, plugins_dir):
|
||||
self.plugins_dir = Path(plugins_dir)
|
||||
self.plugins_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _plugin_dir(self, name: str) -> Path:
|
||||
return self.plugins_dir / name
|
||||
|
||||
# ---- install / uninstall ----
|
||||
|
||||
def install(self, manifest: dict, code: str) -> dict[str, Any]:
|
||||
"""Install a plugin from a manifest dict + entry point source string."""
|
||||
m = PluginManifest.from_dict(manifest)
|
||||
|
||||
plugin_dir = self._plugin_dir(m.name)
|
||||
if plugin_dir.exists():
|
||||
raise ValueError(f"Plugin '{m.name}' is already installed")
|
||||
|
||||
installed = list(self.plugins_dir.iterdir()) if self.plugins_dir.exists() else []
|
||||
if installed and len(installed) >= MAX_PLUGINS_PER_VAULT:
|
||||
raise ValueError(f"Maximum of {MAX_PLUGINS_PER_VAULT} plugins per vault reached")
|
||||
|
||||
if len(code.encode("utf-8")) > MAX_PLUGIN_SIZE:
|
||||
raise ValueError(f"Plugin file exceeds {MAX_PLUGIN_SIZE} bytes")
|
||||
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / MANIFEST_FILENAME).write_text(
|
||||
json.dumps(m.to_dict(), indent=2), encoding="utf-8"
|
||||
)
|
||||
(plugin_dir / m.main).write_text(code, encoding="utf-8")
|
||||
logger.info("Installed plugin '%s' v%s", m.name, m.version)
|
||||
return {
|
||||
"name": m.name,
|
||||
"version": m.version,
|
||||
"description": m.description,
|
||||
"author": m.author,
|
||||
"main": m.main,
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
def uninstall(self, name: str) -> None:
|
||||
plugin_dir = self._plugin_dir(name)
|
||||
if not plugin_dir.exists():
|
||||
raise ValueError(f"Plugin '{name}' not found")
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(plugin_dir)
|
||||
logger.info("Uninstalled plugin '%s'", name)
|
||||
|
||||
# ---- enable / disable ----
|
||||
|
||||
def enable(self, name: str) -> None:
|
||||
plugin_dir = self._plugin_dir(name)
|
||||
if not plugin_dir.exists():
|
||||
raise ValueError(f"Plugin '{name}' not found")
|
||||
marker = plugin_dir / ".disabled"
|
||||
if marker.exists():
|
||||
marker.unlink()
|
||||
|
||||
def disable(self, name: str) -> None:
|
||||
plugin_dir = self._plugin_dir(name)
|
||||
if not plugin_dir.exists():
|
||||
raise ValueError(f"Plugin '{name}' not found")
|
||||
(plugin_dir / ".disabled").write_text("", encoding="utf-8")
|
||||
|
||||
def is_disabled(self, name: str) -> bool:
|
||||
return (self._plugin_dir(name) / ".disabled").exists()
|
||||
|
||||
# ---- listing / metadata ----
|
||||
|
||||
def list_plugins(self) -> list[dict[str, Any]]:
|
||||
if not self.plugins_dir.exists():
|
||||
return []
|
||||
out = []
|
||||
for d in sorted(self.plugins_dir.iterdir()):
|
||||
if not d.is_dir():
|
||||
continue
|
||||
manifest_file = d / MANIFEST_FILENAME
|
||||
if not manifest_file.exists():
|
||||
continue
|
||||
try:
|
||||
m = PluginManifest.from_dict(json.loads(manifest_file.read_text(encoding="utf-8")))
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.warning("Invalid plugin dir '%s': %s", d.name, exc)
|
||||
continue
|
||||
out.append({
|
||||
"name": m.name,
|
||||
"version": m.version,
|
||||
"description": m.description,
|
||||
"author": m.author,
|
||||
"main": m.main,
|
||||
"enabled": not self.is_disabled(m.name),
|
||||
"hooks": m.hooks,
|
||||
"permissions": m.permissions,
|
||||
})
|
||||
return out
|
||||
|
||||
def get_plugin(self, name: str) -> dict[str, Any] | None:
|
||||
plugin_dir = self._plugin_dir(name)
|
||||
manifest_file = plugin_dir / MANIFEST_FILENAME
|
||||
if not manifest_file.exists():
|
||||
return None
|
||||
try:
|
||||
m = PluginManifest.from_dict(json.loads(manifest_file.read_text(encoding="utf-8")))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
return {
|
||||
"manifest": m.to_dict(),
|
||||
"name": m.name,
|
||||
"version": m.version,
|
||||
"enabled": not self.is_disabled(m.name),
|
||||
}
|
||||
|
||||
def get_plugin_code(self, name: str, file_name: str) -> str:
|
||||
plugin_dir = self._plugin_dir(name)
|
||||
manifest_file = plugin_dir / MANIFEST_FILENAME
|
||||
if not manifest_file.exists():
|
||||
raise ValueError(f"Plugin '{name}' not found")
|
||||
target = plugin_dir / file_name
|
||||
# Prevent path traversal
|
||||
try:
|
||||
target.resolve().relative_to(plugin_dir.resolve())
|
||||
except (ValueError, OSError):
|
||||
raise ValueError(f"Invalid file path: {file_name}")
|
||||
if not target.is_file():
|
||||
raise ValueError(f"File '{file_name}' not found for plugin '{name}'")
|
||||
return target.read_text(encoding="utf-8")
|
||||
|
||||
def get_hooks(self, name: str) -> dict[str, str]:
|
||||
plugin = self.get_plugin(name)
|
||||
if not plugin:
|
||||
raise ValueError(f"Plugin '{name}' not found")
|
||||
return plugin["manifest"].get("hooks", {})
|
||||
|
||||
|
||||
class PluginRegistry:
|
||||
"""Aggregates enabled plugins for a plugins directory."""
|
||||
|
||||
def get_enabled_plugins(self, plugins_dir) -> list[dict[str, Any]]:
|
||||
manager = PluginManager(plugins_dir)
|
||||
return [p for p in manager.list_plugins() if p["enabled"]]
|
||||
|
||||
def get_plugins_by_hook(self, plugins_dir, hook: str) -> list[dict[str, Any]]:
|
||||
manager = PluginManager(plugins_dir)
|
||||
return [
|
||||
p for p in manager.list_plugins()
|
||||
if p["enabled"] and hook in p.get("hooks", {})
|
||||
]
|
||||
|
||||
def scan_vault(self, vault_name: str, vault_path) -> list[dict[str, Any]]:
|
||||
"""Scan a vault directory for installed plugins (startup discovery).
|
||||
|
||||
Returns the list of installed-but-valid plugins so the app can log
|
||||
how many are available per vault. Invalid/zombie plugin dirs are
|
||||
skipped gracefully.
|
||||
"""
|
||||
pd = Path(vault_path) / PLUGINS_DIR_NAME
|
||||
if not pd.is_dir():
|
||||
return []
|
||||
return PluginManager(pd).list_plugins()
|
||||
|
||||
|
||||
# Module-level registry singleton (startup scan + runtime lookups)
|
||||
_registry_singleton: PluginRegistry | None = None
|
||||
|
||||
|
||||
def get_plugin_registry() -> PluginRegistry:
|
||||
"""Return the shared PluginRegistry instance."""
|
||||
global _registry_singleton
|
||||
if _registry_singleton is None:
|
||||
_registry_singleton = PluginRegistry()
|
||||
return _registry_singleton
|
||||
|
||||
|
||||
# ── Validation of uploads (ZIP / directory) ────────────────────────────────
|
||||
|
||||
|
||||
def _plugin_manifest_from_member(zf: zipfile.ZipFile) -> dict[str, Any]:
|
||||
try:
|
||||
raw = zf.read(MANIFEST_FILENAME).decode("utf-8")
|
||||
except KeyError:
|
||||
raise ValueError("Plugin zip is missing plugin.json")
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError("Invalid JSON in plugin.json")
|
||||
m = PluginManifest.from_dict(data)
|
||||
return m.to_dict()
|
||||
|
||||
|
||||
def _validate_plugin_zip(zip_path) -> dict[str, Any]:
|
||||
"""Validate an uploaded plugin zip and return its manifest dict."""
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
members = zf.namelist()
|
||||
|
||||
if len(members) > MAX_PLUGIN_FILES:
|
||||
raise ValueError(f"Plugin contains too many files (max {MAX_PLUGIN_FILES})")
|
||||
|
||||
for member in members:
|
||||
if member.endswith("/"):
|
||||
continue
|
||||
norm = member.replace("\\", "/")
|
||||
clean = norm.lstrip("/")
|
||||
if ".." in clean.split("/"):
|
||||
raise ValueError("Path traversal detected in plugin zip")
|
||||
# Reject absolute paths and any ../../ escapes
|
||||
if norm.startswith("/") or "/../" in f"/{norm}" or norm.endswith("/.."):
|
||||
raise ValueError("Path traversal detected in plugin zip")
|
||||
|
||||
manifest = _plugin_manifest_from_member(zf)
|
||||
|
||||
# Entry point must exist
|
||||
main = manifest["main"]
|
||||
if main not in members and f"{main}/" not in members:
|
||||
# accept with any leading ./ or subdir normalization
|
||||
found = any(m.rstrip("/") == main or m.rstrip("/") == f"./{main}" for m in members)
|
||||
if not found:
|
||||
raise ValueError(f"Missing entry point: {main}")
|
||||
return manifest
|
||||
|
||||
|
||||
def _validate_plugin_directory(plugin_dir) -> dict[str, Any]:
|
||||
"""Validate an installed plugin directory and return its manifest dict."""
|
||||
d = Path(plugin_dir)
|
||||
manifest_file = d / MANIFEST_FILENAME
|
||||
if not manifest_file.is_file():
|
||||
raise ValueError(f"Missing {MANIFEST_FILENAME}")
|
||||
try:
|
||||
data = json.loads(manifest_file.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError(f"Invalid JSON in {MANIFEST_FILENAME}")
|
||||
m = PluginManifest.from_dict(data)
|
||||
if not (d / m.main).is_file():
|
||||
raise ValueError(f"Missing entry point: {m.main}")
|
||||
return m.to_dict()
|
||||
|
||||
|
||||
# ── API Endpoints ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("", response_model=list[dict])
|
||||
async def api_list_plugins(vault: str, current_user=Depends(require_auth)):
|
||||
"""List installed plugins for a vault."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
return PluginManager(plugins_dir).list_plugins()
|
||||
|
||||
|
||||
|
||||
@router.get("/template")
|
||||
async def api_plugin_template():
|
||||
"""Return a starter plugin template (manifest + sample code)."""
|
||||
template_manifest = {
|
||||
"name": "my-plugin",
|
||||
"version": "1.0.0",
|
||||
"description": "Describe what your plugin does",
|
||||
"author": "your-name",
|
||||
"main": "index.js",
|
||||
"hooks": {"onFileRender": "render"},
|
||||
"permissions": ["read_files", "ui_notify"],
|
||||
"license": "MIT",
|
||||
}
|
||||
template_code = (
|
||||
"// ObsiGate plugin template\n"
|
||||
"export function render(ctx) {\n"
|
||||
" // ctx: { path, content, extension, vault }\n"
|
||||
" if (ctx.content && ctx.path.endsWith('.md')) {\n"
|
||||
" ctx.content = `> Plugin: ${self.name}\\n\\n` + ctx.content;\n"
|
||||
" }\n"
|
||||
" return ctx;\n"
|
||||
"}\n"
|
||||
)
|
||||
return {
|
||||
"manifest": template_manifest,
|
||||
"code": template_code,
|
||||
}
|
||||
|
||||
@router.get("/{plugin_name}", response_model=dict)
|
||||
async def api_get_plugin(vault: str, plugin_name: str, current_user=Depends(require_auth)):
|
||||
"""Get a single plugin's metadata."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
plugin = PluginManager(plugins_dir).get_plugin(plugin_name)
|
||||
if not plugin:
|
||||
raise HTTPException(status_code=404, detail=f"Plugin '{plugin_name}' not found")
|
||||
return plugin
|
||||
|
||||
|
||||
@router.get("/{plugin_name}/hooks", response_model=dict)
|
||||
async def api_plugin_hooks(vault: str, plugin_name: str, current_user=Depends(require_auth)):
|
||||
"""Get the hooks a plugin registers."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
try:
|
||||
return PluginManager(plugins_dir).get_hooks(plugin_name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
|
||||
|
||||
@router.get("/{plugin_name}/code/{file_name:path}", response_model=dict)
|
||||
async def api_plugin_code(
|
||||
vault: str, plugin_name: str, file_name: str, current_user=Depends(require_auth)
|
||||
):
|
||||
"""Get a plugin source file for sandboxed execution."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
try:
|
||||
code = PluginManager(plugins_dir).get_plugin_code(plugin_name, file_name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return {"name": plugin_name, "file": file_name, "code": code}
|
||||
|
||||
|
||||
@router.post("/install")
|
||||
async def api_install_plugin(
|
||||
vault: str,
|
||||
file: UploadFile = File(...),
|
||||
current_user=Depends(require_admin),
|
||||
):
|
||||
"""Install a plugin from an uploaded zip (admin only)."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
|
||||
content = await file.read()
|
||||
if len(content) > 5_000_000: # 5 MB upload cap
|
||||
raise HTTPException(status_code=413, detail="Plugin zip too large")
|
||||
|
||||
import io
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
zip_path = Path(tmp_dir) / "plugin.zip"
|
||||
try:
|
||||
zip_path.write_bytes(content)
|
||||
manifest = _validate_plugin_zip(zip_path)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid plugin: {exc}")
|
||||
|
||||
# Extract to a temp dir then validate the entry point exists
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
manager = PluginManager(plugins_dir)
|
||||
|
||||
name = manifest["name"]
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||||
zf.extractall(td)
|
||||
except (zipfile.BadZipFile, RuntimeError):
|
||||
raise HTTPException(status_code=400, detail="Invalid plugin zip")
|
||||
extracted = _validate_plugin_directory(Path(td) / name if (Path(td) / name).is_dir() else Path(td))
|
||||
|
||||
code_file = manifest["main"]
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||||
code = zf.read(code_file).decode("utf-8")
|
||||
|
||||
try:
|
||||
result = manager.install(extracted, code)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/{plugin_name}")
|
||||
async def api_uninstall_plugin(vault: str, plugin_name: str, current_user=Depends(require_admin)):
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
try:
|
||||
PluginManager(plugins_dir).uninstall(plugin_name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{plugin_name}/enable")
|
||||
async def api_enable_plugin(vault: str, plugin_name: str, current_user=Depends(require_admin)):
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
try:
|
||||
PluginManager(plugins_dir).enable(plugin_name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{plugin_name}/disable")
|
||||
async def api_disable_plugin(vault: str, plugin_name: str, current_user=Depends(require_admin)):
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault}'")
|
||||
plugins_dir = _vault_plugins_dir(vault)
|
||||
try:
|
||||
PluginManager(plugins_dir).disable(plugin_name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
# ── Plugin Event Bus ────────────────────────────────────────────────────────
|
||||
# Lightweight pub/sub for backend events that plugins can subscribe to.
|
||||
# Used to dispatch onFileCreate, onFileDelete, onVaultMount to enabled plugins.
|
||||
|
||||
|
||||
class _PluginEventBus:
|
||||
"""In-memory event dispatcher for plugin lifecycle events."""
|
||||
_listeners: ClassVar[dict[str, list]] = {}
|
||||
|
||||
@classmethod
|
||||
def subscribe(cls, event_type: str, callback) -> None:
|
||||
cls._listeners.setdefault(event_type, []).append(callback)
|
||||
|
||||
@classmethod
|
||||
async def emit(cls, event_type: str, data: dict) -> None:
|
||||
for cb in cls._listeners.get(event_type, []):
|
||||
try:
|
||||
result = cb(data)
|
||||
if hasattr(result, "__await__"):
|
||||
await result
|
||||
except Exception as e:
|
||||
logger.warning(f"Plugin event '{event_type}' handler error: {e}")
|
||||
|
||||
|
||||
plugin_events = _PluginEventBus
|
||||
|
||||
|
||||
def emit_file_created(vault: str, path: str, title: str = "") -> None:
|
||||
"""Called when a file is created via the API."""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(plugin_events.emit("onFileCreate", {
|
||||
"vault": vault, "path": path, "title": title,
|
||||
"timestamp": _time.time(),
|
||||
}))
|
||||
except RuntimeError:
|
||||
pass # No event loop running
|
||||
|
||||
|
||||
def emit_file_deleted(vault: str, path: str) -> None:
|
||||
"""Called when a file is deleted via the API."""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(plugin_events.emit("onFileDelete", {
|
||||
"vault": vault, "path": path,
|
||||
"timestamp": _time.time(),
|
||||
}))
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def emit_vault_mounted(vault: str, path: str) -> None:
|
||||
"""Called when a vault is loaded into the index at startup."""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(plugin_events.emit("onVaultMount", {
|
||||
"vault": vault, "path": path,
|
||||
"timestamp": _time.time(),
|
||||
}))
|
||||
except RuntimeError:
|
||||
pass
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Provider-declared model capabilities (live) with a short-lived cache.
|
||||
|
||||
The curated table in :mod:`backend.model_capabilities` has to be edited by hand
|
||||
every time a provider ships or renames a model, and it ages badly: Mistral alone
|
||||
declares ``vision`` on 28 of its models while the curated table knew none of
|
||||
them (BUG-044). Providers that *do* publish per-model capabilities in their
|
||||
public models endpoint are therefore asked first; the curated table is only
|
||||
used to fill the flags the provider stays silent about (e.g. Mistral never
|
||||
declares ``embedding``, only the absence of ``completion_chat``).
|
||||
|
||||
Supported declarations, detected by *payload shape* so a provider that starts
|
||||
exposing them is picked up without a code change:
|
||||
|
||||
* ``capabilities`` dict (Mistral): ``completion_chat`` → ``chat``, ``vision``,
|
||||
``audio_transcription`` (+ ``audio_transcription_realtime``),
|
||||
``audio_speech``.
|
||||
* ``architecture`` dict (OpenRouter): ``input_modalities`` /
|
||||
``output_modalities`` → ``vision`` (image input), ``images`` (image output),
|
||||
``audio_transcription`` (audio input), ``audio_speech`` (audio output),
|
||||
``video`` (video output), ``chat`` (text output).
|
||||
|
||||
The cache is in-process and shared by every request. Its TTL only bounds how
|
||||
long a *stale* declaration can survive: a fresh provider call (the model list
|
||||
endpoint) overwrites the provider entry immediately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.ai.capabilities")
|
||||
|
||||
#: How long a provider-declared snapshot stays usable, in seconds.
|
||||
#: ``0`` disables expiry (the snapshot lives until the next provider call).
|
||||
TTL_SECONDS = float(os.getenv("AI_CAPABILITIES_TTL_SECONDS", "1800") or 0)
|
||||
|
||||
#: Guard against a provider returning a runaway model list.
|
||||
MAX_MODELS_PER_PROVIDER = 2000
|
||||
|
||||
# provider → (timestamp, model count, {normalized model id: declared flags})
|
||||
_CACHE: dict[str, tuple[float, int, dict[str, dict[str, bool]]]] = {}
|
||||
|
||||
|
||||
def _norm(value: str) -> str:
|
||||
"""Lower-case and strip the ``models/`` prefix Gemini uses."""
|
||||
return (value or "").strip().lower().removeprefix("models/")
|
||||
|
||||
|
||||
def _from_capability_flags(flags: dict[str, Any]) -> dict[str, bool]:
|
||||
"""Map a Mistral-style ``capabilities`` dict onto ObsiGate flags."""
|
||||
declared: dict[str, bool] = {}
|
||||
if isinstance(flags.get("completion_chat"), bool):
|
||||
declared["chat"] = flags["completion_chat"]
|
||||
if isinstance(flags.get("vision"), bool):
|
||||
declared["vision"] = flags["vision"]
|
||||
transcription = flags.get("audio_transcription")
|
||||
realtime = flags.get("audio_transcription_realtime")
|
||||
if isinstance(transcription, bool) or isinstance(realtime, bool):
|
||||
declared["audio_transcription"] = bool(transcription or realtime)
|
||||
if isinstance(flags.get("audio_speech"), bool):
|
||||
declared["audio_speech"] = flags["audio_speech"]
|
||||
return declared
|
||||
|
||||
|
||||
def _from_architecture(architecture: dict[str, Any]) -> dict[str, bool]:
|
||||
"""Map an OpenRouter-style ``architecture`` dict onto ObsiGate flags."""
|
||||
inputs = architecture.get("input_modalities")
|
||||
outputs = architecture.get("output_modalities")
|
||||
if not isinstance(inputs, list) and not isinstance(outputs, list):
|
||||
return {}
|
||||
in_modalities = [str(m).lower() for m in inputs] if isinstance(inputs, list) else []
|
||||
out_modalities = [str(m).lower() for m in outputs] if isinstance(outputs, list) else []
|
||||
return {
|
||||
"chat": "text" in out_modalities,
|
||||
"vision": "image" in in_modalities,
|
||||
"images": "image" in out_modalities,
|
||||
"audio_transcription": "audio" in in_modalities,
|
||||
"audio_speech": "audio" in out_modalities,
|
||||
"video": "video" in out_modalities,
|
||||
}
|
||||
|
||||
|
||||
def parse_declared_capabilities(entry: Any) -> dict[str, bool] | None:
|
||||
"""Extract the capability flags a single provider model entry declares.
|
||||
|
||||
Args:
|
||||
entry: One item of a provider models payload (``/v1/models``).
|
||||
|
||||
Returns:
|
||||
A partial ``{flag: bool}`` mapping (only the flags the provider
|
||||
actually declares), or ``None`` when the entry declares nothing.
|
||||
"""
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
flags = entry.get("capabilities")
|
||||
declared = _from_capability_flags(flags) if isinstance(flags, dict) else {}
|
||||
if not declared:
|
||||
architecture = entry.get("architecture")
|
||||
declared = _from_architecture(architecture) if isinstance(architecture, dict) else {}
|
||||
return declared or None
|
||||
|
||||
|
||||
def remember_declared_capabilities(provider: str, payload: Any) -> int:
|
||||
"""Cache the capabilities declared by a provider models payload.
|
||||
|
||||
Args:
|
||||
provider: Provider identifier (e.g. ``"mistral"``).
|
||||
payload: Raw JSON body of the provider models endpoint, or the model
|
||||
list itself.
|
||||
|
||||
Returns:
|
||||
The number of models with declared capabilities that were cached.
|
||||
"""
|
||||
provider = (provider or "").strip().lower()
|
||||
entries: Any = payload.get("data") if isinstance(payload, dict) else payload
|
||||
if not isinstance(entries, list):
|
||||
return 0
|
||||
|
||||
parsed: dict[str, dict[str, bool]] = {}
|
||||
for entry in entries[:MAX_MODELS_PER_PROVIDER]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
model_id = entry.get("id") or entry.get("name") or ""
|
||||
if not isinstance(model_id, str) or not model_id.strip():
|
||||
continue
|
||||
declared = parse_declared_capabilities(entry)
|
||||
if declared:
|
||||
parsed[_norm(model_id)] = declared
|
||||
|
||||
if not parsed:
|
||||
return 0
|
||||
_CACHE[provider] = (time.time(), len(parsed), parsed)
|
||||
logger.info(f"Capabilities declared by {provider}: {len(parsed)} models cached")
|
||||
return len(parsed)
|
||||
|
||||
|
||||
def get_declared_capabilities(provider: str, model: str) -> dict[str, bool] | None:
|
||||
"""Return the cached declared capabilities for one provider/model pair.
|
||||
|
||||
Returns ``None`` when nothing was declared for that pair (cache cold,
|
||||
expired, or the provider is silent about this model).
|
||||
"""
|
||||
snapshot = _CACHE.get((provider or "").strip().lower())
|
||||
if not snapshot:
|
||||
return None
|
||||
timestamp, _count, table = snapshot
|
||||
if TTL_SECONDS and (time.time() - timestamp) > TTL_SECONDS:
|
||||
return None
|
||||
declared = table.get(_norm(model))
|
||||
return dict(declared) if declared else None
|
||||
|
||||
|
||||
def clear_declared_capabilities(provider: str | None = None) -> None:
|
||||
"""Drop the cached snapshot for one provider, or all of them (tests/ops)."""
|
||||
if provider is None:
|
||||
_CACHE.clear()
|
||||
return
|
||||
_CACHE.pop(provider.strip().lower(), None)
|
||||
|
||||
|
||||
def cache_info() -> dict[str, dict[str, Any]]:
|
||||
"""Diagnostics: per-provider cache age and model count."""
|
||||
now = time.time()
|
||||
return {
|
||||
provider: {
|
||||
"models": count,
|
||||
"age_seconds": round(now - timestamp, 1),
|
||||
"expired": bool(TTL_SECONDS and (now - timestamp) > TTL_SECONDS),
|
||||
}
|
||||
for provider, (timestamp, count, _table) in _CACHE.items()
|
||||
}
|
||||
+285
@@ -0,0 +1,285 @@
|
||||
# backend/push.py
|
||||
# Push Notifications — Web Push API with VAPID authentication
|
||||
# ROADMAP #67
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
|
||||
logger = logging.getLogger("obsigate.push")
|
||||
|
||||
router = APIRouter(prefix="/api/push", tags=["push"])
|
||||
|
||||
# VAPID keys storage
|
||||
VAPID_KEYS_FILE = Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / "vapid_keys.json"
|
||||
PUSH_SUBSCRIPTIONS_FILE = Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / "push_subscriptions.json"
|
||||
|
||||
# In-memory cache
|
||||
_vapid_keys: dict[str, str] = {}
|
||||
_push_subscriptions: list[dict[str, Any]] = []
|
||||
|
||||
|
||||
def load_vapid_keys() -> dict[str, str]:
|
||||
"""Load VAPID keys from file or generate new ones."""
|
||||
global _vapid_keys
|
||||
if _vapid_keys:
|
||||
return _vapid_keys
|
||||
|
||||
try:
|
||||
if VAPID_KEYS_FILE.exists():
|
||||
with open(VAPID_KEYS_FILE, "r") as f:
|
||||
_vapid_keys = json.load(f)
|
||||
else:
|
||||
# Generate new VAPID keys
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
|
||||
private_key = ec.generate_private_key(ec.SECP256R1())
|
||||
public_key = private_key.public_key()
|
||||
|
||||
private_pem = private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
).decode('utf-8')
|
||||
|
||||
public_pem = public_key.public_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PublicFormat.SubjectPublicKeyInfo
|
||||
).decode('utf-8')
|
||||
|
||||
# Convert to base64url for Web Push
|
||||
public_numbers = public_key.public_numbers()
|
||||
|
||||
def int_to_base64url(n: int) -> str:
|
||||
byte_len = (n.bit_length() + 7) // 8
|
||||
return base64.urlsafe_b64encode(n.to_bytes(byte_len, 'big')).decode('utf-8').rstrip('=')
|
||||
|
||||
_vapid_keys = {
|
||||
"private_key": private_pem,
|
||||
"public_key": public_pem,
|
||||
"public_key_base64url": int_to_base64url(public_numbers.x) + "." + int_to_base64url(public_numbers.y)
|
||||
}
|
||||
|
||||
VAPID_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(VAPID_KEYS_FILE, "w") as f:
|
||||
json.dump(_vapid_keys, f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load/generate VAPID keys: {e}")
|
||||
# Fallback: return empty (will use demo keys if needed)
|
||||
_vapid_keys = {}
|
||||
|
||||
return _vapid_keys
|
||||
|
||||
|
||||
def get_vapid_public_key() -> str:
|
||||
"""Get VAPID public key in base64url format for client."""
|
||||
keys = load_vapid_keys()
|
||||
return keys.get("public_key_base64url", "")
|
||||
|
||||
|
||||
def load_push_subscriptions() -> list[dict[str, Any]]:
|
||||
"""Load push subscriptions from file."""
|
||||
global _push_subscriptions
|
||||
if _push_subscriptions:
|
||||
return _push_subscriptions
|
||||
|
||||
try:
|
||||
if PUSH_SUBSCRIPTIONS_FILE.exists():
|
||||
with open(PUSH_SUBSCRIPTIONS_FILE, "r") as f:
|
||||
_push_subscriptions = json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load push subscriptions: {e}")
|
||||
_push_subscriptions = []
|
||||
return _push_subscriptions
|
||||
|
||||
|
||||
def save_push_subscriptions() -> None:
|
||||
"""Save push subscriptions to file."""
|
||||
try:
|
||||
PUSH_SUBSCRIPTIONS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(PUSH_SUBSCRIPTIONS_FILE, "w") as f:
|
||||
json.dump(_push_subscriptions, f, indent=2)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save push subscriptions: {e}")
|
||||
|
||||
|
||||
# ── Models ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class PushSubscription(BaseModel):
|
||||
"""Push subscription from client (matches Push API)."""
|
||||
endpoint: str
|
||||
keys: dict[str, str] # { p256dh, auth }
|
||||
|
||||
|
||||
class SubscribeRequest(BaseModel):
|
||||
"""Request to subscribe to push notifications."""
|
||||
subscription: PushSubscription
|
||||
vault: str = Field(description="Vault name this subscription is for")
|
||||
|
||||
|
||||
class SubscribeResponse(BaseModel):
|
||||
"""Response to subscription request."""
|
||||
success: bool
|
||||
subscription_id: str | None = None
|
||||
|
||||
|
||||
class VapidPublicKeyResponse(BaseModel):
|
||||
"""VAPID public key for client."""
|
||||
public_key: str
|
||||
|
||||
|
||||
class PushPayload(BaseModel):
|
||||
"""Payload for sending a push notification."""
|
||||
vault: str
|
||||
title: str
|
||||
body: str
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
tag: str = "obsigate-notification"
|
||||
|
||||
|
||||
# ── Endpoints ───────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/vapid-public-key", response_model=VapidPublicKeyResponse)
|
||||
async def get_vapid_public_key_endpoint():
|
||||
"""Get VAPID public key for client subscription."""
|
||||
public_key = get_vapid_public_key()
|
||||
if not public_key:
|
||||
# Return a demo key if generation failed (for testing)
|
||||
return {"public_key": "demo-key-for-testing"}
|
||||
return {"public_key": public_key}
|
||||
|
||||
|
||||
@router.post("/subscribe", response_model=SubscribeResponse)
|
||||
async def subscribe_push(
|
||||
request: SubscribeRequest,
|
||||
current_user=Depends(require_auth)
|
||||
):
|
||||
"""Subscribe to push notifications for a vault."""
|
||||
username = current_user.get("username", "unknown")
|
||||
|
||||
# Check if user has access to this vault (#194 : "*" n'inclut pas les
|
||||
# dossiers persos, check_vault_access est la source unique).
|
||||
if not check_vault_access(request.vault, current_user):
|
||||
raise HTTPException(status_code=403, detail="No access to this vault")
|
||||
|
||||
# Check if subscription already exists
|
||||
for sub in _push_subscriptions:
|
||||
if sub["endpoint"] == request.subscription.endpoint and sub["username"] == username:
|
||||
return SubscribeResponse(success=True, subscription_id=sub.get("id"))
|
||||
|
||||
# Add new subscription
|
||||
sub_id = base64.urlsafe_b64encode(os.urandom(16)).decode('utf-8').rstrip('=')
|
||||
subscription = {
|
||||
"id": sub_id,
|
||||
"endpoint": request.subscription.endpoint,
|
||||
"keys": request.subscription.keys,
|
||||
"vault": request.vault,
|
||||
"username": username,
|
||||
"created_at": datetime.now(UTC).isoformat()
|
||||
}
|
||||
|
||||
_push_subscriptions.append(subscription)
|
||||
save_push_subscriptions()
|
||||
|
||||
logger.info(f"Push subscription added for user={username}, vault={request.vault}")
|
||||
return SubscribeResponse(success=True, subscription_id=sub_id)
|
||||
|
||||
|
||||
@router.delete("/subscribe")
|
||||
async def unsubscribe_push(
|
||||
endpoint: str,
|
||||
current_user=Depends(require_auth)
|
||||
):
|
||||
"""Unsubscribe from push notifications."""
|
||||
username = current_user.get("username", "unknown")
|
||||
|
||||
global _push_subscriptions
|
||||
original_len = len(_push_subscriptions)
|
||||
_push_subscriptions = [
|
||||
s for s in _push_subscriptions
|
||||
if not (s["endpoint"] == endpoint and s["username"] == username)
|
||||
]
|
||||
|
||||
if len(_push_subscriptions) < original_len:
|
||||
save_push_subscriptions()
|
||||
return {"success": True, "message": "Unsubscribed"}
|
||||
|
||||
return {"success": False, "message": "Subscription not found"}
|
||||
|
||||
|
||||
@router.get("/subscriptions")
|
||||
async def list_subscriptions(current_user=Depends(require_auth)):
|
||||
"""List current user's push subscriptions."""
|
||||
username = current_user.get("username", "unknown")
|
||||
user_subs = [
|
||||
{
|
||||
"id": s["id"],
|
||||
"vault": s["vault"],
|
||||
"created_at": s["created_at"],
|
||||
"endpoint": s["endpoint"][:50] + "..." # Truncate for privacy
|
||||
}
|
||||
for s in _push_subscriptions if s["username"] == username
|
||||
]
|
||||
return {"subscriptions": user_subs}
|
||||
|
||||
|
||||
# ── Internal: Send push notification ────────────────────────────────────────
|
||||
|
||||
async def send_push_notification(vault: str, title: str, body: str, data: dict | None = None, tag: str = "obsigate-notification") -> int:
|
||||
"""Send push notification to all subscribers of a vault. Returns count of sent notifications."""
|
||||
subscriptions = [s for s in _push_subscriptions if s["vault"] == vault]
|
||||
|
||||
if not subscriptions:
|
||||
return 0
|
||||
|
||||
payload = {
|
||||
"title": title,
|
||||
"body": body,
|
||||
"data": data or {},
|
||||
"tag": tag
|
||||
}
|
||||
|
||||
# Import here to avoid circular dependency
|
||||
from pywebpush import WebPushException, webpush
|
||||
|
||||
keys = load_vapid_keys()
|
||||
vapid_private_key = keys.get("private_key")
|
||||
vapid_claims = {
|
||||
"sub": "mailto:[email protected]"
|
||||
}
|
||||
|
||||
sent = 0
|
||||
for sub in subscriptions:
|
||||
try:
|
||||
webpush(
|
||||
subscription_info={
|
||||
"endpoint": sub["endpoint"],
|
||||
"keys": sub["keys"]
|
||||
},
|
||||
data=json.dumps(payload),
|
||||
vapid_private_key=vapid_private_key,
|
||||
vapid_claims=vapid_claims
|
||||
)
|
||||
sent += 1
|
||||
except WebPushException as e:
|
||||
logger.warning(f"Push failed for subscription {sub['id']}: {e}")
|
||||
# If subscription expired/invalid, remove it
|
||||
if e.response and e.response.status_code in (404, 410):
|
||||
_push_subscriptions.remove(sub)
|
||||
except Exception as e:
|
||||
logger.error(f"Push error for {sub['id']}: {e}")
|
||||
|
||||
if sent < len(subscriptions):
|
||||
save_push_subscriptions()
|
||||
|
||||
return sent
|
||||
+235
-15
@@ -1,17 +1,35 @@
|
||||
"""
|
||||
IP-based rate limiter for authentication endpoints.
|
||||
In-memory rate limiter for authentication endpoints.
|
||||
|
||||
Tracks failed login attempts per IP address with automatic
|
||||
cleanup of expired entries. Complements the per-account lockout
|
||||
in user_store.py.
|
||||
Tracks failed attempts per IP **and** per account with automatic cleanup of
|
||||
expired entries. The per-IP budget stops a single source; the per-account
|
||||
budget (BUG-031) still throttles an attacker who rotates IPs. It complements
|
||||
the per-account lockout in ``user_store.py``.
|
||||
|
||||
.. note::
|
||||
The counters live in process memory only. They are **not** shared between
|
||||
multiple workers/containers and are lost on restart. For a multi-node
|
||||
deployment, front this service with a shared store (Redis) or a single
|
||||
worker. This limitation is intentional and documented (BUG-031).
|
||||
|
||||
Opt-in persistence (ROADMAP #85 T10b) : if ``OBSIGATE_RATELIMIT_DB`` points
|
||||
to a SQLite file, counters are stored there instead (WAL mode, one short
|
||||
connection per call — safe across threads, processes and restarts sharing
|
||||
the same file). Semantics (windows, budgets, success reset) are identical
|
||||
to the in-memory store, which remains the default when the variable is
|
||||
unset.
|
||||
|
||||
Configuration via environment variables:
|
||||
OBSIGATE_LOGIN_MAX_ATTEMPTS Max failures per IP (default: 10)
|
||||
OBSIGATE_LOGIN_WINDOW_SECONDS Lockout window in seconds (default: 900 = 15min)
|
||||
OBSIGATE_LOGIN_MAX_ATTEMPTS Max failures per IP (default: 10)
|
||||
OBSIGATE_ACCOUNT_MAX_ATTEMPTS Max failures per account (default: 10)
|
||||
OBSIGATE_LOGIN_WINDOW_SECONDS Lockout window in seconds (default: 900)
|
||||
OBSIGATE_RATELIMIT_DB SQLite file for shared/persistent counters (default: unset = memory)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
@@ -19,29 +37,158 @@ logger = logging.getLogger("obsigate.ratelimit")
|
||||
|
||||
# --- Configuration ---
|
||||
MAX_ATTEMPTS = int(os.environ.get("OBSIGATE_LOGIN_MAX_ATTEMPTS", "10"))
|
||||
ACCOUNT_MAX_ATTEMPTS = int(os.environ.get("OBSIGATE_ACCOUNT_MAX_ATTEMPTS", "10"))
|
||||
WINDOW_SECONDS = int(os.environ.get("OBSIGATE_LOGIN_WINDOW_SECONDS", "900")) # 15 min
|
||||
|
||||
# --- In-memory store: {ip: [(timestamp, success_bool), ...]} ---
|
||||
# --- In-memory stores: {key: [(timestamp, success_bool), ...]} ---
|
||||
_ip_attempts: dict[str, list] = defaultdict(list)
|
||||
_account_attempts: dict[str, list] = defaultdict(list)
|
||||
_last_cleanup = time.time()
|
||||
CLEANUP_INTERVAL = 60 # seconds
|
||||
|
||||
|
||||
def _db_path() -> str | None:
|
||||
"""SQLite file for shared counters, or ``None`` for the in-memory store."""
|
||||
path = os.environ.get("OBSIGATE_RATELIMIT_DB", "").strip()
|
||||
return path or None
|
||||
|
||||
|
||||
def _db_connect(path: str) -> sqlite3.Connection:
|
||||
"""Open a short-lived connection (WAL + busy timeout for concurrent workers)."""
|
||||
_db_ensure_schema(path)
|
||||
conn = sqlite3.connect(path, timeout=10.0)
|
||||
conn.execute("PRAGMA busy_timeout=10000")
|
||||
return conn
|
||||
|
||||
|
||||
_schema_ready: set[str] = set()
|
||||
_schema_lock = threading.Lock()
|
||||
|
||||
|
||||
def _db_ensure_schema(path: str) -> None:
|
||||
"""Create the store schema once per file (DDL under a process-wide lock)."""
|
||||
with _schema_lock:
|
||||
if path in _schema_ready:
|
||||
return
|
||||
conn = sqlite3.connect(path, timeout=10.0)
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS attempts"
|
||||
" (kind TEXT NOT NULL, key TEXT NOT NULL, ts REAL NOT NULL, success INTEGER NOT NULL)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_attempts_kind_key_ts"
|
||||
" ON attempts (kind, key, ts)"
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
_schema_ready.add(path)
|
||||
|
||||
|
||||
def _db_write(fn, *args):
|
||||
"""Run a write op, retrying once on lock contention (concurrent workers)."""
|
||||
try:
|
||||
return fn(*args)
|
||||
except sqlite3.OperationalError as e:
|
||||
if "locked" not in str(e).lower():
|
||||
raise
|
||||
time.sleep(0.05)
|
||||
return fn(*args)
|
||||
|
||||
|
||||
def _db_prune(conn: sqlite3.Connection, cutoff: float) -> None:
|
||||
"""Drop expired entries (best-effort cap on disk growth)."""
|
||||
conn.execute("DELETE FROM attempts WHERE ts <= ?", (cutoff,))
|
||||
|
||||
|
||||
def _db_record(kind: str, key: str, success: bool) -> int:
|
||||
"""Record one attempt in SQLite; return the live failure count."""
|
||||
path = _db_path()
|
||||
assert path is not None
|
||||
now = time.time()
|
||||
cutoff = now - WINDOW_SECONDS
|
||||
|
||||
def _write() -> int:
|
||||
with _db_connect(path) as conn:
|
||||
_db_prune(conn, cutoff)
|
||||
if success:
|
||||
# Mirror the in-memory reset: replace history with one success.
|
||||
conn.execute("DELETE FROM attempts WHERE kind = ? AND key = ?", (kind, key))
|
||||
conn.execute(
|
||||
"INSERT INTO attempts (kind, key, ts, success) VALUES (?, ?, ?, ?)",
|
||||
(kind, key, now, int(success)),
|
||||
)
|
||||
conn.commit()
|
||||
(failures,) = conn.execute(
|
||||
"SELECT COUNT(*) FROM attempts WHERE kind = ? AND key = ? AND ts > ? AND success = 0",
|
||||
(kind, key, cutoff),
|
||||
).fetchone()
|
||||
return failures
|
||||
|
||||
return _db_write(_write)
|
||||
|
||||
|
||||
def _db_failures(kind: str, key: str) -> int:
|
||||
"""Live failure count in SQLite (expired entries never count)."""
|
||||
path = _db_path()
|
||||
assert path is not None
|
||||
cutoff = time.time() - WINDOW_SECONDS
|
||||
with _db_connect(path) as conn:
|
||||
(failures,) = conn.execute(
|
||||
"SELECT COUNT(*) FROM attempts WHERE kind = ? AND key = ? AND ts > ? AND success = 0",
|
||||
(kind, key, cutoff),
|
||||
).fetchone()
|
||||
return failures
|
||||
|
||||
|
||||
def _db_tracked(kind: str) -> int:
|
||||
"""Number of distinct keys ever seen for one budget (SQLite)."""
|
||||
path = _db_path()
|
||||
assert path is not None
|
||||
with _db_connect(path) as conn:
|
||||
(n,) = conn.execute(
|
||||
"SELECT COUNT(DISTINCT key) FROM attempts WHERE kind = ?", (kind,)
|
||||
).fetchone()
|
||||
return n
|
||||
|
||||
|
||||
def _db_limited_count(kind: str, max_attempts: int) -> int:
|
||||
"""Number of keys currently over budget (SQLite)."""
|
||||
path = _db_path()
|
||||
assert path is not None
|
||||
cutoff = time.time() - WINDOW_SECONDS
|
||||
with _db_connect(path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT key, COUNT(*) FROM attempts"
|
||||
" WHERE kind = ? AND ts > ? AND success = 0 GROUP BY key",
|
||||
(kind, cutoff),
|
||||
).fetchall()
|
||||
return sum(1 for _, n in rows if n >= max_attempts)
|
||||
|
||||
|
||||
def _prune(store: dict[str, list], cutoff: float) -> None:
|
||||
"""Drop expired entries from one store in place."""
|
||||
expired = []
|
||||
for key, attempts in store.items():
|
||||
store[key] = [a for a in attempts if a[0] > cutoff]
|
||||
if not store[key]:
|
||||
expired.append(key)
|
||||
for key in expired:
|
||||
del store[key]
|
||||
|
||||
|
||||
def _cleanup_expired():
|
||||
"""Remove entries older than the window."""
|
||||
"""Remove entries older than the window from both stores."""
|
||||
global _last_cleanup
|
||||
now = time.time()
|
||||
if now - _last_cleanup < CLEANUP_INTERVAL:
|
||||
return
|
||||
_last_cleanup = now
|
||||
cutoff = now - WINDOW_SECONDS
|
||||
expired_ips = []
|
||||
for ip, attempts in _ip_attempts.items():
|
||||
_ip_attempts[ip] = [a for a in attempts if a[0] > cutoff]
|
||||
if not _ip_attempts[ip]:
|
||||
expired_ips.append(ip)
|
||||
for ip in expired_ips:
|
||||
del _ip_attempts[ip]
|
||||
_prune(_ip_attempts, cutoff)
|
||||
_prune(_account_attempts, cutoff)
|
||||
|
||||
|
||||
def record_failure(ip: str) -> tuple[int, int]:
|
||||
@@ -50,6 +197,12 @@ def record_failure(ip: str) -> tuple[int, int]:
|
||||
Returns:
|
||||
(current_failure_count, remaining_attempts)
|
||||
"""
|
||||
if _db_path() is not None:
|
||||
failures = _db_record("ip", ip, False)
|
||||
remaining = max(0, MAX_ATTEMPTS - failures)
|
||||
if failures >= MAX_ATTEMPTS:
|
||||
logger.warning(f"IP {ip} rate-limited after {failures} failed logins")
|
||||
return failures, remaining
|
||||
_cleanup_expired()
|
||||
_ip_attempts[ip].append((time.time(), False))
|
||||
failures = sum(1 for _, success in _ip_attempts[ip] if not success)
|
||||
@@ -61,19 +214,84 @@ def record_failure(ip: str) -> tuple[int, int]:
|
||||
|
||||
def record_success(ip: str):
|
||||
"""Clear rate limit state for an IP after successful login."""
|
||||
if _db_path() is not None:
|
||||
_db_record("ip", ip, True)
|
||||
return
|
||||
_cleanup_expired()
|
||||
_ip_attempts[ip] = [(time.time(), True)]
|
||||
|
||||
|
||||
def is_rate_limited(ip: str) -> bool:
|
||||
"""Check if an IP has exceeded the rate limit."""
|
||||
if _db_path() is not None:
|
||||
return _db_failures("ip", ip) >= MAX_ATTEMPTS
|
||||
_cleanup_expired()
|
||||
failures = sum(1 for _, success in _ip_attempts.get(ip, []) if not success)
|
||||
return failures >= MAX_ATTEMPTS
|
||||
|
||||
|
||||
def record_account_failure(account: str) -> tuple[int, int]:
|
||||
"""Record a failed attempt for an account, regardless of source IP.
|
||||
|
||||
Returns:
|
||||
(current_failure_count, remaining_attempts)
|
||||
"""
|
||||
key = account.lower()
|
||||
if _db_path() is not None:
|
||||
failures = _db_record("account", key, False)
|
||||
remaining = max(0, ACCOUNT_MAX_ATTEMPTS - failures)
|
||||
if failures >= ACCOUNT_MAX_ATTEMPTS:
|
||||
logger.warning(f"Account {account} rate-limited after {failures} failed attempts")
|
||||
return failures, remaining
|
||||
_cleanup_expired()
|
||||
_account_attempts[key].append((time.time(), False))
|
||||
failures = sum(1 for _, success in _account_attempts[key] if not success)
|
||||
remaining = max(0, ACCOUNT_MAX_ATTEMPTS - failures)
|
||||
if failures >= ACCOUNT_MAX_ATTEMPTS:
|
||||
logger.warning(f"Account {account} rate-limited after {failures} failed attempts")
|
||||
return failures, remaining
|
||||
|
||||
|
||||
def record_account_success(account: str):
|
||||
"""Clear the per-account rate limit state after a successful login."""
|
||||
if _db_path() is not None:
|
||||
_db_record("account", account.lower(), True)
|
||||
return
|
||||
_cleanup_expired()
|
||||
_account_attempts[account.lower()] = [(time.time(), True)]
|
||||
|
||||
|
||||
def is_account_rate_limited(account: str) -> bool:
|
||||
"""Check if an account has exceeded the per-account rate limit."""
|
||||
if _db_path() is not None:
|
||||
return _db_failures("account", account.lower()) >= ACCOUNT_MAX_ATTEMPTS
|
||||
_cleanup_expired()
|
||||
failures = sum(
|
||||
1 for _, success in _account_attempts.get(account.lower(), []) if not success
|
||||
)
|
||||
return failures >= ACCOUNT_MAX_ATTEMPTS
|
||||
|
||||
|
||||
def get_status(ip: str | None = None) -> dict:
|
||||
"""Get rate limit status for an IP (for diagnostics)."""
|
||||
if _db_path() is not None:
|
||||
if ip:
|
||||
failures = _db_failures("ip", ip)
|
||||
return {
|
||||
"ip": ip,
|
||||
"failures": failures,
|
||||
"max": MAX_ATTEMPTS,
|
||||
"limited": failures >= MAX_ATTEMPTS,
|
||||
"window_seconds": WINDOW_SECONDS,
|
||||
}
|
||||
return {
|
||||
"tracked_ips": _db_tracked("ip"),
|
||||
"tracked_accounts": _db_tracked("account"),
|
||||
"max_attempts": MAX_ATTEMPTS,
|
||||
"account_max_attempts": ACCOUNT_MAX_ATTEMPTS,
|
||||
"window_seconds": WINDOW_SECONDS,
|
||||
"limited_ips": _db_limited_count("ip", MAX_ATTEMPTS),
|
||||
}
|
||||
_cleanup_expired()
|
||||
if ip:
|
||||
attempts = _ip_attempts.get(ip, [])
|
||||
@@ -87,7 +305,9 @@ def get_status(ip: str | None = None) -> dict:
|
||||
}
|
||||
return {
|
||||
"tracked_ips": len(_ip_attempts),
|
||||
"tracked_accounts": len(_account_attempts),
|
||||
"max_attempts": MAX_ATTEMPTS,
|
||||
"account_max_attempts": ACCOUNT_MAX_ATTEMPTS,
|
||||
"window_seconds": WINDOW_SECONDS,
|
||||
"limited_ips": sum(
|
||||
1 for ip_addr in _ip_attempts
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""CORS bornée pour la recherche fédérée (fusion-trois-apps #68).
|
||||
|
||||
La page « Rechercher partout » (``/recherche``) appelle, depuis le
|
||||
navigateur, ``GET /api/search`` sur chaque application sœur. Sans
|
||||
en-têtes CORS, le navigateur refuse la lecture des réponses
|
||||
cross-origin ; sans borne, on ouvrirait la lecture à tout le web.
|
||||
Ce module n'autorise donc que :
|
||||
|
||||
- le chemin ``/api/search`` (jamais les autres routes, jamais les
|
||||
écritures) ;
|
||||
- les origines sœurs : les origines officielles **prod ET dev,
|
||||
listées explicitement** (``PROD_ORIGINS | DEV_ORIGINS``), **plus
|
||||
les origines de la console Spark** (``AGENT_ORIGINS`` : agent prod
|
||||
ET dev — fédération de la recherche depuis la console,
|
||||
2026-10-28), **plus**
|
||||
celles de ``backend.draco_config`` (``urls_publiques()`` :
|
||||
explicite > dérivation par l'hôte de l'agent > défaut officiel)
|
||||
— jamais ``*``. Correctif 2026-10-28 (fusion #68/#69, matrice
|
||||
constatée par Bruno) : l'ancienne allowlist ne contenait que
|
||||
l'environnement de la configuration (plus celui de l'hôte de la
|
||||
requête), jamais les deux à la fois — une page de l'autre
|
||||
environnement voyait sa section afficher « Injoignable depuis
|
||||
cet hôte (CORS ou réseau) » ;
|
||||
- avec credentials (le cookie SSO ``draco_session``, accepté par
|
||||
``/api/search`` via le repli de ``backend.auth.middleware``, ou
|
||||
le cookie natif ``access_token``).
|
||||
|
||||
Jamais ``*``. Le middleware est un ASGI pur : il répond lui-même
|
||||
aux preflights OPTIONS (qui ne portent jamais de cookies) et ajoute
|
||||
les en-têtes aux réponses de recherche, sans toucher aux autres
|
||||
routes ni à la politique same-origin explicite du CORSMiddleware
|
||||
global (``allow_origins=[]``), qui reste la règle par défaut.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
SEARCH_PATH = "/api/search"
|
||||
|
||||
PROD_ORIGINS = frozenset({
|
||||
"https://og.dracodev.net",
|
||||
"https://flowdeck.dracodev.net",
|
||||
"https://tiktak.dracodev.net",
|
||||
"https://kanbante.dracodev.net",
|
||||
"https://shaarit.dracodev.net",
|
||||
})
|
||||
|
||||
DEV_ORIGINS = frozenset({
|
||||
"https://og-dev.dracodev.net",
|
||||
"https://flowdeck-dev.dracodev.net",
|
||||
"https://tiktak-dev.dracodev.net",
|
||||
"https://kanbante-dev.dracodev.net",
|
||||
"https://shaarit-dev.dracodev.net",
|
||||
})
|
||||
|
||||
AGENT_ORIGINS = frozenset({
|
||||
"https://agent.dracodev.net",
|
||||
"https://agent-dev.dracodev.net",
|
||||
})
|
||||
|
||||
|
||||
# Clés de draco_config.urls_publiques() qui sont des applications sœurs.
|
||||
_SOEUR_KEYS = ("obsigate", "flowdeck", "tiktak", "kanbante", "shaarit")
|
||||
|
||||
|
||||
def is_dev_host(hostname: str | None) -> bool:
|
||||
"""Même règle que la barre v0.1.6 (voir frontend/js/recherche.js)."""
|
||||
if not hostname:
|
||||
return False
|
||||
h = hostname.lower().split(":")[0]
|
||||
if not h.endswith(".dracodev.net"):
|
||||
return False
|
||||
first = h.split(".")[0]
|
||||
return len(first) > 4 and first.endswith("-dev")
|
||||
|
||||
|
||||
def is_prod_host(hostname: str | None) -> bool:
|
||||
"""Hôte dracodev.net qui n'est ni ``-dev`` ni un piège de suffixe."""
|
||||
if not hostname:
|
||||
return False
|
||||
h = hostname.lower().split(":")[0]
|
||||
if not h.endswith(".dracodev.net"):
|
||||
return False
|
||||
first = h.split(".")[0]
|
||||
return len(first) > 0 and not first.endswith("-dev")
|
||||
|
||||
|
||||
def origin_of(url: str | None) -> str | None:
|
||||
"""Origine (schéma + hôte) d'une URL, sans chemin ni ``/`` final."""
|
||||
if not url:
|
||||
return None
|
||||
parsed = urlparse(url.strip())
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
return None
|
||||
host = parsed.hostname.lower()
|
||||
port = f":{parsed.port}" if parsed.port else ""
|
||||
return f"{parsed.scheme}://{host}{port}"
|
||||
|
||||
|
||||
def sister_origins(resolved: dict | None = None, request_host: str | None = None) -> frozenset:
|
||||
"""Origines autorisées pour la lecture cross-origin de /api/search.
|
||||
|
||||
``resolved`` : dictionnaire rendu par ``urls_publiques()``.
|
||||
Sans argument, le résolveur est appelé (import paresseux) ; en cas
|
||||
d'échec, repli sur prod + dev officiels (premier parti uniquement).
|
||||
``request_host`` ajoute l'ensemble officiel de l'environnement de
|
||||
la requête (voir le docstring du module).
|
||||
"""
|
||||
origins = set()
|
||||
if resolved is None:
|
||||
try:
|
||||
from backend.draco_config import urls_publiques
|
||||
|
||||
resolved = urls_publiques()
|
||||
except Exception:
|
||||
resolved = {}
|
||||
for key in _SOEUR_KEYS:
|
||||
origin = origin_of((resolved or {}).get(key))
|
||||
if origin:
|
||||
origins.add(origin)
|
||||
# Correctif 2026-10-28 : prod ET dev toujours — ``request_host``
|
||||
# ne restreint plus l'allowlist (il ne servait qu'à ajouter
|
||||
# l'environnement de la requête, désormais inclus d'office).
|
||||
# ``resolved`` n'ajoute que des origines de configuration sans
|
||||
# jamais retirer une origine officielle.
|
||||
origins |= PROD_ORIGINS | DEV_ORIGINS | AGENT_ORIGINS
|
||||
return frozenset(origins)
|
||||
|
||||
|
||||
def cors_headers_for(
|
||||
origin: str | None, resolved: dict | None = None, request_host: str | None = None
|
||||
) -> dict | None:
|
||||
"""En-têtes CORS d'une réponse de recherche, ou None si refusée."""
|
||||
if not origin:
|
||||
return None
|
||||
if origin.strip() not in sister_origins(resolved, request_host):
|
||||
return None
|
||||
return {
|
||||
"Access-Control-Allow-Origin": origin.strip(),
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
"Vary": "Origin",
|
||||
}
|
||||
|
||||
|
||||
def preflight_headers(
|
||||
origin: str | None, resolved: dict | None = None, request_host: str | None = None
|
||||
) -> dict | None:
|
||||
"""En-têtes d'une réponse de preflight OPTIONS pour /api/search."""
|
||||
headers = cors_headers_for(origin, resolved, request_host)
|
||||
if headers is None:
|
||||
return None
|
||||
headers.update({
|
||||
"Access-Control-Allow-Methods": "GET, OPTIONS",
|
||||
"Access-Control-Allow-Headers": "Accept, Content-Type, Authorization",
|
||||
"Access-Control-Max-Age": "600",
|
||||
})
|
||||
return headers
|
||||
|
||||
|
||||
class RechercheCorsMiddleware:
|
||||
"""ASGI pur : CORS bornée pour ``GET /api/search`` (voir module)."""
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope.get("type") != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
path = scope.get("path", "")
|
||||
headers_in = {
|
||||
k.decode("latin-1").lower(): v.decode("latin-1")
|
||||
for k, v in scope.get("headers", [])
|
||||
}
|
||||
origin = headers_in.get("origin", "")
|
||||
request_host = (
|
||||
headers_in.get("x-forwarded-host", "") or headers_in.get("host", "")
|
||||
).split(",")[0].strip()
|
||||
if path == SEARCH_PATH and scope.get("method") == "OPTIONS":
|
||||
preflight = preflight_headers(origin, request_host=request_host)
|
||||
if preflight is not None:
|
||||
headers_out = [(k.lower().encode("latin-1"), v.encode("latin-1"))
|
||||
for k, v in preflight.items()]
|
||||
headers_out.append((b"content-length", b"0"))
|
||||
await send({"type": "http.response.start", "status": 204,
|
||||
"headers": headers_out})
|
||||
await send({"type": "http.response.body", "body": b""})
|
||||
return
|
||||
if path != SEARCH_PATH:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
async def send_with_cors(message):
|
||||
if message["type"] == "http.response.start":
|
||||
cors = cors_headers_for(origin, request_host=request_host)
|
||||
if cors is not None:
|
||||
message["headers"] = list(message.get("headers", [])) + [
|
||||
(k.lower().encode("latin-1"), v.encode("latin-1"))
|
||||
for k, v in cors.items()
|
||||
]
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, receive, send_with_cors)
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Markdown rendering pipeline (ROADMAP #85, tranche 9).
|
||||
|
||||
Helpers extraits de :mod:`backend.main` sans changement de comportement :
|
||||
slugification des headings, IDs d'ancrage, rendu mistune singleton,
|
||||
wikilinks, normalisation des sauts de ligne et pipeline complet
|
||||
:func:`_render_markdown` (rendu + sanitizer XSS BUG-021).
|
||||
|
||||
Les noms gardent leur préfixe ``_`` d'origine pour un déplacement
|
||||
strictement verbatim (tests et routers pointent ici désormais).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_mod
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import mistune
|
||||
|
||||
from backend.image_processor import preprocess_images
|
||||
from backend.indexer import find_file_in_index, get_vault_data
|
||||
from backend.secret_redactor import redact_with_placeholders, restore_masks
|
||||
from backend.services.sanitizer import sanitize_html
|
||||
|
||||
|
||||
def _heading_slugify(text: str) -> str:
|
||||
"""Generate a URL-safe slug from heading text.
|
||||
|
||||
Matches the JavaScript slugify algorithm exactly using
|
||||
Unicode-aware character classification:
|
||||
1. Strip HTML tags (e.g. wikilink spans rendered inside headings)
|
||||
2. Decode HTML entities (e.g. ``&`` → ``&``)
|
||||
3. Lowercase
|
||||
4. NFD normalize + strip combining marks
|
||||
5. Keep only Unicode letters, numbers, spaces, hyphens
|
||||
6. Replace spaces with hyphens, collapse multiple hyphens
|
||||
|
||||
Args:
|
||||
text: The heading text content (may contain inline HTML).
|
||||
|
||||
Returns:
|
||||
A URL-safe slug string.
|
||||
"""
|
||||
# Strip any inline HTML so it does not pollute the slug
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
# Decode HTML entities so & becomes & before slugification
|
||||
text = html_mod.unescape(text)
|
||||
text = text.lower()
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
||||
# Unicode-aware: keep letters (L*), numbers (N*), spaces, and hyphens
|
||||
cleaned = []
|
||||
for ch in text:
|
||||
cat = unicodedata.category(ch)
|
||||
if cat.startswith('L') or cat.startswith('N') or ch in (' ', '-'):
|
||||
cleaned.append(ch)
|
||||
text = "".join(cleaned)
|
||||
text = re.sub(r"\s+", "-", text)
|
||||
text = re.sub(r"-+", "-", text)
|
||||
result = text.strip("-")
|
||||
return result if result else "heading"
|
||||
|
||||
|
||||
def _add_heading_ids(html: str) -> str:
|
||||
"""Post-process rendered HTML to add IDs to heading tags.
|
||||
|
||||
Adds an ``id`` attribute to every ``<h1>`` through ``<h6>`` tag
|
||||
using a slug generated from the heading's text content.
|
||||
Duplicate slugs get a ``-2``, ``-3``, etc. suffix.
|
||||
|
||||
Args:
|
||||
html: Rendered HTML string.
|
||||
|
||||
Returns:
|
||||
HTML with heading IDs injected.
|
||||
"""
|
||||
used_ids: dict[str, int] = {}
|
||||
|
||||
def _replace_heading(match):
|
||||
tag = match.group(1)
|
||||
content = match.group(2)
|
||||
slug = _heading_slugify(content)
|
||||
count = used_ids.get(slug, 0)
|
||||
used_ids[slug] = count + 1
|
||||
if count > 0:
|
||||
slug = f"{slug}-{count + 1}"
|
||||
return f'<{tag} id="{slug}">{content}</{tag}>'
|
||||
|
||||
# Match h1-h6 tags with text content (no existing id attribute)
|
||||
return re.sub(
|
||||
r'<(h[1-6])>([^<]*(?:<(?!/?h[1-6])[^<]*)*)</h[1-6]>',
|
||||
_replace_heading,
|
||||
html,
|
||||
)
|
||||
|
||||
|
||||
# Cached mistune renderer — avoids re-creating on every request
|
||||
_markdown_renderer = mistune.create_markdown(
|
||||
escape=False,
|
||||
plugins=["table", "strikethrough", "footnotes", "task_lists"],
|
||||
)
|
||||
|
||||
|
||||
def _convert_wikilinks(content: str, current_vault: str) -> str:
|
||||
"""Convert ``[[wikilinks]]`` and ``[[target|display]]`` to clickable HTML.
|
||||
|
||||
Supports:
|
||||
- Internal file links: ``[[My Note]]`` / ``[[My Note|display]]``
|
||||
- Same-document anchors: ``[[#Heading]]`` / ``[[#Heading|display]]``
|
||||
|
||||
Resolved file links get a ``data-vault`` / ``data-path`` attribute pair.
|
||||
Anchor links target the slugified heading ID in the current document.
|
||||
Unresolved links are rendered as ``<span class="wikilink-missing">``.
|
||||
|
||||
Args:
|
||||
content: Markdown string potentially containing wikilinks.
|
||||
current_vault: Active vault name for resolution priority.
|
||||
|
||||
Returns:
|
||||
Markdown string with wikilinks replaced by HTML anchors.
|
||||
"""
|
||||
def _replace(match):
|
||||
target = match.group(1).strip()
|
||||
display = match.group(2).strip() if match.group(2) else target
|
||||
|
||||
# Same-document anchor link: [[#Heading|display]]
|
||||
if target.startswith("#"):
|
||||
anchor_text = target[1:].strip()
|
||||
anchor_slug = _heading_slugify(anchor_text)
|
||||
link_display = display if display != target else anchor_text
|
||||
return f'<a class="wikilink-anchor" href="#{anchor_slug}">{link_display}</a>'
|
||||
|
||||
found = find_file_in_index(target, current_vault)
|
||||
if found:
|
||||
return (
|
||||
f'<a class="wikilink" href="#" '
|
||||
f'data-vault="{found["vault"]}" '
|
||||
f'data-path="{found["path"]}">{display}</a>'
|
||||
)
|
||||
return f'<span class="wikilink-missing">{display}</span>'
|
||||
|
||||
pattern = r'\[\[([^\]|]+)(?:\|([^\]]+))?\]\]'
|
||||
return re.sub(pattern, _replace, content)
|
||||
|
||||
|
||||
def _normalize_line_breaks(text: str) -> str:
|
||||
"""Convert single newlines to hard breaks (matching Obsidian default behavior).
|
||||
|
||||
In standard Markdown, a single ``\\n`` is a "soft break" — it renders as a space,
|
||||
not a visible line break. Obsidian defaults to treating single newlines as hard
|
||||
breaks (equivalent to ``<br>``). This function pre-processes the Markdown source
|
||||
so that mistune renders standalone lines on separate rows, while still honouring
|
||||
blank lines as paragraph separators.
|
||||
|
||||
Fenced code blocks (`` ``` ``) are left untouched so their internal newlines are
|
||||
preserved verbatim.
|
||||
"""
|
||||
parts = re.split(r"(```[\s\S]*?```)", text)
|
||||
for i, part in enumerate(parts):
|
||||
if part.startswith("```"):
|
||||
continue # Protect fenced code blocks
|
||||
# Single \n (not preceded or followed by another \n) → two spaces + \n
|
||||
parts[i] = re.sub(r"(?<!\n)\n(?!\n)", " \n", part)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _render_markdown(
|
||||
raw_md: str,
|
||||
vault_name: str,
|
||||
current_file_path: Path | None = None,
|
||||
*,
|
||||
click_to_copy: bool = False,
|
||||
) -> str:
|
||||
"""Render a markdown string to HTML with wikilink and image support.
|
||||
|
||||
Uses the cached singleton mistune renderer for performance.
|
||||
|
||||
Args:
|
||||
raw_md: Raw markdown text (frontmatter already stripped).
|
||||
vault_name: Current vault for wikilink resolution context.
|
||||
current_file_path: Absolute path to the current markdown file.
|
||||
click_to_copy: Restore masked secrets as clickable badges carrying
|
||||
the real value (authenticated app preview, feature #188).
|
||||
Public shares and PDF exports keep plain labels: the secret
|
||||
never reaches their HTML.
|
||||
|
||||
Returns:
|
||||
HTML string.
|
||||
"""
|
||||
# Get vault data for image resolution
|
||||
vault_data = get_vault_data(vault_name)
|
||||
vault_root = Path(vault_data["path"]) if vault_data else None
|
||||
attachments_path = vault_data.get("config", {}).get("attachmentsPath") if vault_data else None
|
||||
|
||||
# Redact secrets before rendering (P0 security). Placeholders survive
|
||||
# the markdown conversion (fenced code blocks included) and are turned
|
||||
# back into visible masks — clickable badges when click_to_copy — right
|
||||
# after the HTML is produced (feature #188).
|
||||
raw_md, secret_entries = redact_with_placeholders(
|
||||
raw_md, str(current_file_path) if current_file_path else ""
|
||||
)
|
||||
|
||||
# Preprocess images first
|
||||
if vault_root:
|
||||
raw_md = preprocess_images(raw_md, vault_name, vault_root, current_file_path, attachments_path)
|
||||
|
||||
# Convert wikilinks
|
||||
converted = _convert_wikilinks(raw_md, vault_name)
|
||||
|
||||
# Normalize line breaks to match Obsidian behavior (single \n → hard break)
|
||||
converted = _normalize_line_breaks(converted)
|
||||
|
||||
# mistune 3.3 types `Markdown.__call__` as `str | list[...]` (les
|
||||
# renderers HTML renvoient toujours `str` à l'exécution).
|
||||
rendered = cast(str, _markdown_renderer(converted))
|
||||
|
||||
# Restore secret masks (plain labels, or clickable badges carrying the
|
||||
# real value on the authenticated app preview — feature #188).
|
||||
rendered = restore_masks(rendered, secret_entries, click_to_copy=click_to_copy)
|
||||
|
||||
# Add heading IDs for TOC navigation
|
||||
rendered = _add_heading_ids(rendered)
|
||||
|
||||
# Sanitize: raw HTML in vault content must never reach the DOM (BUG-021).
|
||||
rendered = sanitize_html(rendered)
|
||||
|
||||
return rendered
|
||||
@@ -0,0 +1,18 @@
|
||||
# ObsiGate — Optional dependencies for semantic search (#70)
|
||||
#
|
||||
# These are NOT required: the semantic search engine degrades gracefully to a
|
||||
# dependency-free hashing embedder and a pure-Python cosine store when they are
|
||||
# absent. Install this file to enable the full local model + fast vector index:
|
||||
#
|
||||
# pip install -r backend/requirements-semantic.txt
|
||||
#
|
||||
# NOTE: sentence-transformers pulls in PyTorch (large download). If you only
|
||||
# want the vector acceleration, install numpy + faiss-cpu and configure an
|
||||
# external embedding endpoint instead (OBSIGATE_EMBEDDING_*).
|
||||
|
||||
# Local embedding model (all-MiniLM-L6-v2, ~80 MB, CPU)
|
||||
sentence-transformers>=2.2.0
|
||||
|
||||
# Vector storage / similarity search
|
||||
numpy>=1.24.0
|
||||
faiss-cpu>=1.7.4
|
||||
@@ -0,0 +1,5 @@
|
||||
# Additional dev/test-only dependencies for the test suite.
|
||||
# These are NOT required for production runtime.
|
||||
# Install with: pip install -r requirements-test.txt
|
||||
|
||||
reportlab>=4.0 # PDF fixture generation for test_pdf.py
|
||||
@@ -1,15 +1,89 @@
|
||||
fastapi==0.110.3
|
||||
uvicorn==0.30.0
|
||||
fastapi==0.141.1
|
||||
uvicorn==0.54.0
|
||||
websockets>=12.0
|
||||
python-frontmatter==1.1.0
|
||||
mistune==3.0.2
|
||||
python-multipart==0.0.9
|
||||
mistune==3.3.3
|
||||
python-multipart==0.0.31
|
||||
aiofiles==23.2.1
|
||||
aiohttp>=3.9.0
|
||||
# Plancher de sécurité : 3.14.3 est vulnérable à 8 CVE (dont
|
||||
# CVE-2026-111063/111064 et GHSA-2g87-pp6c-29x5), toutes corrigées en
|
||||
# 3.14.4 — pip-audit (CI, bloquant) exige de rester au-dessus.
|
||||
aiohttp>=3.14.4
|
||||
watchdog>=4.0.0
|
||||
argon2-cffi>=23.1.0
|
||||
python-jose>=3.3.0
|
||||
sortedcontainers>=2.4.0
|
||||
snowballstemmer>=2.2.0
|
||||
weasyprint>=60.0
|
||||
weasyprint>=70.0
|
||||
httpx>=0.27.0
|
||||
pypdf>=4.0
|
||||
# Plancher de sécurité (BUG-093, relevé par #224) : 6.16.0 est vulnérable à
|
||||
# deux DoS de ressources (PYSEC-2026-3910 outlines, PYSEC-2026-3911 XForm,
|
||||
# fix 6.16.1), atteignables via backend/pdf_reader.py (PDF fournis par
|
||||
# l'utilisateur). #224 : la version préinstallée dans la toolcache du
|
||||
# runner (6.19.0) est frappée par 11 CVE supplémentaires (dont
|
||||
# CVE-2026-113786, CVE-2026-115572, CVE-2026-115585, CVE-2026-115580),
|
||||
# corrigées au plus tard en 6.21.2. La 6.21.2 est à son tour frappée
|
||||
# (CVE-2026-119203 et suivantes, relevées par le job security sur main),
|
||||
# corrigées en 6.21.4 — le plancher doit être >= 6.21.4 :
|
||||
# en dessous, pip répond « already satisfied » pour la toolcache et
|
||||
# n'aligne jamais (même piège que BUG-093).
|
||||
pypdf>=6.21.4
|
||||
pyotp>=2.10.0
|
||||
segno>=1.5.0
|
||||
webauthn==2.6.0
|
||||
psutil>=5.9
|
||||
pywebpush>=2.3.0
|
||||
# 1.28.1 vulnérable (CVE-2026-59951, fix 1.30.0) — épinglé sur la
|
||||
# première version corrigée de la même ligne 1.x.
|
||||
mcp==1.30.0
|
||||
# Plancher de sécurité (BUG-091, BUG-095) : pyjwt est une dépendance transitive
|
||||
# (mcp). 2.12.x → PYSEC-2026-178 (fix 2.13.0) ; 2.13.0 → CVE-2026-102274
|
||||
# (fix 2.14.0). pip-audit étant bloquant, on reste au-dessus du dernier correctif.
|
||||
pyjwt[crypto]>=2.14.0
|
||||
sse-starlette==2.1.3
|
||||
openpyxl>=3.1
|
||||
xlrd==2.0.2
|
||||
odfpy==1.4.1
|
||||
python-docx>=1.1
|
||||
reportlab>=4.0
|
||||
# Plancher de sécurité : 12.3.0 est vulnérable à 8 CVE (dont
|
||||
# CVE-2026-112895 et CVE-2026-112400), toutes corrigées en 13.0.0.
|
||||
pillow>=13.0.0
|
||||
# Plancher urllib3 >= 2.8.0 (CVE-2026-97687, CVE-2026-97688, CVE-2026-97689)
|
||||
urllib3>=2.8.0
|
||||
# Plancher de sécurité (CVE-2026-104874, fix 6.9.1) : multidict est transitive
|
||||
# (aiohttp/yarl). 6.7.x est la version pré-installée dans la toolcache de
|
||||
# l'image du runner — sans plancher, pip répond « already satisfied » et
|
||||
# n'aligne jamais (même piège que pypdf, BUG-093).
|
||||
multidict>=6.9.1
|
||||
# Plancher de sécurité (PYSEC-2026-4320, fix 2026.10.19) : certifi est
|
||||
# transitive (httpx/requests) — sans plancher explicite, pip-audit
|
||||
# signale la version héritée de l'environnement.
|
||||
certifi>=2026.10.19
|
||||
# Planchers de sécurité (#224) : paquets transitifs ou préinstallés dans
|
||||
# la toolcache du runner, audités par pip-audit (CI, bloquant) mais jamais
|
||||
# alignés sans plancher explicite — « already satisfied » (même piège que
|
||||
# BUG-093). Versions relevées par le job security du run sur 2.78.0 :
|
||||
# brotli 1.2.0 (transitive fonttools[woff] via weasyprint) — PYSEC-2026-4322
|
||||
# et GHSA-9v8h-qqgm-4m35, corrigées en 1.2.1.
|
||||
brotli>=1.2.1
|
||||
# filelock 3.32.3 (transitive de CacheControl, outil pip-audit, et
|
||||
# préinstallée dans la toolcache) — CVE-2026-112940, CVE-2026-112913 et
|
||||
# CVE-2026-112907, corrigées en 4.2.1.
|
||||
filelock>=4.2.1
|
||||
# opentelemetry-api 1.37.0 (préinstallée dans la toolcache du runner,
|
||||
# aucune dépendance directe d'ObsiGate) — CVE-2026-114603, corrigée
|
||||
# en 1.45.1.
|
||||
opentelemetry-api>=1.45.1
|
||||
# Coffre Secrets (Phase 1a) : chiffrement par enveloppe AES-256-GCM.
|
||||
cryptography>=44.0
|
||||
|
||||
# Plancher de securite : msgpack est transitif, version 1.2.1 relevee par le job
|
||||
# security (pip-audit, bloquant) — CVE-2026-114613, corrigee en 1.2.3.
|
||||
msgpack>=1.2.3
|
||||
# Plancher de sécurité : pyasn1 est transitive (python-jose/rsa), version
|
||||
# 0.6.4 relevée par le job security (pip-audit, bloquant) —
|
||||
# CVE-2026-119194 à CVE-2026-119197, corrigées en 0.6.5. Sans plancher
|
||||
# explicite, pip répond « already satisfied » pour la toolcache et
|
||||
# n'aligne jamais (même piège que BUG-093).
|
||||
pyasn1>=0.6.5
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""ObsiGate — routers FastAPI par domaine (ROADMAP #85).
|
||||
|
||||
Découpage progressif du monolithe ``backend/main.py`` : chaque module de ce
|
||||
paquet expose un ``APIRouter`` monté par ``main.py``. Les handlers sont
|
||||
déplacés sans changement de comportement (mêmes chemins, mêmes modèles de
|
||||
réponse, mêmes dépendances d'authentification).
|
||||
"""
|
||||
@@ -0,0 +1,412 @@
|
||||
"""Backup endpoints (ROADMAP #85, tranche 4).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/file/{vault}/backups|diff|restore``,
|
||||
``/api/backups*``), mêmes modèles de réponse, mêmes dépendances
|
||||
d'authentification. La logique métier vit déjà dans
|
||||
:mod:`backend.services.backups`.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_resolve_safe_path`` / ``_backup_file`` / ``_list_backup_files`` de
|
||||
``main`` n'étaient que des wrappers directs : appelés ici via
|
||||
:mod:`backend.services.paths` et :mod:`backend.services.backups`.
|
||||
- ``RestoreRequest`` / ``RestoreResponse`` / ``DiffResponse`` ont déménagé
|
||||
dans :mod:`backend.schemas`.
|
||||
- Le singleton SSE vit désormais dans :mod:`backend.sse` (partagé avec
|
||||
``main`` : les clients ``/api/events`` reçoivent les mêmes broadcasts).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.indexer import get_vault_data, index, update_single_file
|
||||
from backend.schemas import (
|
||||
BackupContentResponse,
|
||||
BackupsAutoResponse,
|
||||
BackupsCompressResponse,
|
||||
BackupsDeletedResponse,
|
||||
BackupsListResponse,
|
||||
BackupsResponse,
|
||||
DiffResponse,
|
||||
RestoreRequest,
|
||||
RestoreResponse,
|
||||
)
|
||||
from backend.services.backups import (
|
||||
create_backup,
|
||||
)
|
||||
from backend.services.backups import (
|
||||
diff_backup as service_diff_backup,
|
||||
)
|
||||
from backend.services.backups import (
|
||||
list_backup_files as service_list_backup_files,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
restore_backup as service_restore_backup,
|
||||
)
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.sse import sse_manager
|
||||
from backend.webhooks import dispatch_webhooks
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["backups"])
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/backups", response_model=BackupsResponse)
|
||||
async def api_file_backups(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""List all available backups for a file.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path of the file within the vault.
|
||||
|
||||
Returns:
|
||||
BackupListResponse with backups sorted newest first.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
try:
|
||||
backups = service_list_backup_files(vault_name, path)
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing backups for {vault_name}/{path}: {type(e).__name__}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Erreur lors de la lecture des backups: {e!s}")
|
||||
|
||||
return {"vault": vault_name, "path": path, "backups": backups}
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/diff", response_model=DiffResponse)
|
||||
async def api_file_diff(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
version: int = Query(..., description="Timestamp of the backup version (left/old side)"),
|
||||
compare_with: int | None = Query(default=None, description="Timestamp of another backup (right/new side). If omitted, compares with the current file."),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Generate a unified diff between a backup version and another version or the current file.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path of the file within the vault.
|
||||
version: Timestamp of the backup to use as the old/left side.
|
||||
compare_with: Optional timestamp of another backup as the new/right side.
|
||||
If omitted, the current file on disk is used.
|
||||
|
||||
Returns:
|
||||
DiffResponse containing the unified diff string.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
return service_diff_backup(vault_name, path, version, compare_with)
|
||||
|
||||
|
||||
@router.post("/api/file/{vault_name}/restore", response_model=RestoreResponse)
|
||||
async def api_file_restore(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
body: RestoreRequest = ..., # type: ignore
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Restore a file from a backup version.
|
||||
|
||||
The current file is backed up before being overwritten (so the operation is reversible).
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path of the file within the vault.
|
||||
body: RestoreRequest with the backup version timestamp.
|
||||
|
||||
Returns:
|
||||
RestoreResponse confirming the restore.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_restore_backup(vault_name, path, body.version)
|
||||
current_backed_up = result["current_backed_up"]
|
||||
|
||||
# Update index
|
||||
await update_single_file(vault_name, path)
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("file_restored", {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"restored_from": body.version,
|
||||
"current_backed_up": current_backed_up,
|
||||
})
|
||||
await dispatch_webhooks("file_restored", {"vault": vault_name, "path": path, "restored_from": body.version})
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"restored_from": body.version,
|
||||
"current_backed_up": current_backed_up,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/backups", response_model=BackupsListResponse)
|
||||
async def api_backups_list(
|
||||
vault: str | None = Query(None, description="Filter by vault name"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""List all backups across vaults, grouped by file."""
|
||||
result: list[dict[str, Any]] = []
|
||||
try:
|
||||
for vault_name in index:
|
||||
if vault and vault_name != vault:
|
||||
continue
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
continue
|
||||
vd = get_vault_data(vault_name)
|
||||
if not vd:
|
||||
continue
|
||||
vault_root = Path(vd["path"])
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
if not backup_root.is_absolute():
|
||||
backup_root = vault_root / backup_root
|
||||
vault_backup_dir = backup_root / vault_name
|
||||
if not vault_backup_dir.exists():
|
||||
continue
|
||||
for fpath in vault_backup_dir.rglob("*.bak"):
|
||||
if not fpath.is_file():
|
||||
continue
|
||||
st = fpath.stat()
|
||||
fsize = st.st_size
|
||||
ts_part = fpath.name.rsplit(".", 2)
|
||||
if len(ts_part) < 3 or not ts_part[-2].isdigit():
|
||||
continue
|
||||
ts = int(ts_part[-2])
|
||||
rel_dir = str(fpath.parent.relative_to(vault_backup_dir)).replace("\\", "/")
|
||||
rel_file = rel_dir + "/" + ts_part[0] if rel_dir != "." else ts_part[0]
|
||||
result.append({
|
||||
"vault": vault_name,
|
||||
"file": rel_file,
|
||||
"backup_file": fpath.name,
|
||||
"timestamp": ts,
|
||||
"datetime": datetime.fromtimestamp(ts, tz=UTC).isoformat(),
|
||||
"size": fsize,
|
||||
"full_path": str(fpath),
|
||||
})
|
||||
|
||||
result.sort(key=lambda x: x["timestamp"], reverse=True)
|
||||
total_size = sum(r["size"] for r in result)
|
||||
return {"backups": result, "total": len(result), "total_size_bytes": total_size}
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing backups: {type(e).__name__}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Erreur listing backups: {e!s}")
|
||||
|
||||
|
||||
@router.post("/api/backups/delete", response_model=BackupsDeletedResponse)
|
||||
async def api_backups_delete(
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Delete one or more backup files."""
|
||||
paths = body.get("paths", [])
|
||||
if not paths:
|
||||
raise HTTPException(status_code=400, detail="No backup paths provided")
|
||||
|
||||
deleted = 0
|
||||
for p in paths:
|
||||
try:
|
||||
fpath = Path(p)
|
||||
# Security: ensure path is within a backup directory
|
||||
if ".obsigate-backup" not in str(fpath):
|
||||
continue
|
||||
if fpath.exists() and fpath.is_file():
|
||||
fpath.unlink()
|
||||
deleted += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete backup {p}: {e}")
|
||||
|
||||
return {"deleted": deleted}
|
||||
|
||||
|
||||
@router.post("/api/backups/purge", response_model=BackupsDeletedResponse)
|
||||
async def api_backups_purge(
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Purge all backups for a specific file or entire vault."""
|
||||
vault_name = body.get("vault")
|
||||
file_path = body.get("file") # optional
|
||||
|
||||
if not vault_name:
|
||||
raise HTTPException(status_code=400, detail="Vault name required")
|
||||
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail="Access denied")
|
||||
|
||||
vd = get_vault_data(vault_name)
|
||||
if not vd:
|
||||
raise HTTPException(status_code=404, detail="Vault not found")
|
||||
|
||||
vault_root = Path(vd["path"])
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
if not backup_root.is_absolute():
|
||||
backup_root = vault_root / backup_root
|
||||
|
||||
if file_path:
|
||||
# Delete backups for specific file
|
||||
backup_dir = backup_root / vault_name / Path(file_path).parent
|
||||
if backup_dir.exists():
|
||||
fname = Path(file_path).name
|
||||
deleted = 0
|
||||
for f in backup_dir.iterdir():
|
||||
if f.is_file() and f.name.startswith(fname + ".") and f.name.endswith(".bak"):
|
||||
f.unlink()
|
||||
deleted += 1
|
||||
return {"deleted": deleted}
|
||||
return {"deleted": 0}
|
||||
else:
|
||||
# Delete all backups for vault
|
||||
vault_backup_dir = backup_root / vault_name
|
||||
if vault_backup_dir.exists():
|
||||
deleted = 0
|
||||
for f in vault_backup_dir.rglob("*.bak"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
deleted += 1
|
||||
return {"deleted": deleted}
|
||||
return {"deleted": 0}
|
||||
|
||||
|
||||
|
||||
@router.get("/api/backups/content", response_model=BackupContentResponse)
|
||||
async def api_backups_content(
|
||||
path: str = Query(..., description="Full path to backup file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return the content of a specific backup file."""
|
||||
try:
|
||||
fpath = Path(path)
|
||||
if ".obsigate-backup" not in str(fpath):
|
||||
raise HTTPException(status_code=403, detail="Access denied")
|
||||
if not fpath.exists() or not fpath.is_file():
|
||||
raise HTTPException(status_code=404, detail="Backup not found")
|
||||
content = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
# Truncate large files to 100KB
|
||||
if len(content) > 102400:
|
||||
content = content[:102400] + "\n\n... (tronque a 100 Ko)"
|
||||
return {"content": content, "name": fpath.name, "size": len(content)}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/api/backups/compress", response_model=BackupsCompressResponse)
|
||||
async def api_backups_compress(
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Compress backups older than N days. Body: {older_than_days: 30, dry_run: false}"""
|
||||
import gzip as gz_mod
|
||||
older_than = body.get("older_than_days", 30)
|
||||
dry_run = body.get("dry_run", False)
|
||||
cutoff = time.time() - (older_than * 86400)
|
||||
compressed = 0
|
||||
saved_bytes = 0
|
||||
|
||||
for vault_name in index:
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
continue
|
||||
vd = get_vault_data(vault_name)
|
||||
if not vd:
|
||||
continue
|
||||
vault_root = Path(vd["path"])
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
if not backup_root.is_absolute():
|
||||
backup_root = vault_root / backup_root
|
||||
vault_dir = backup_root / vault_name
|
||||
if not vault_dir.exists():
|
||||
continue
|
||||
for fpath in vault_dir.rglob("*.bak"):
|
||||
if not fpath.is_file():
|
||||
continue
|
||||
if fpath.name.endswith(".bak.gz"):
|
||||
continue
|
||||
mtime = fpath.stat().st_mtime
|
||||
if mtime > cutoff:
|
||||
continue
|
||||
if not dry_run:
|
||||
try:
|
||||
gz_path = fpath.with_suffix(fpath.suffix + ".gz")
|
||||
data = fpath.read_bytes()
|
||||
with gz_mod.open(str(gz_path), "wb", compresslevel=6) as gzf:
|
||||
gzf.write(data)
|
||||
orig_size = len(data)
|
||||
gz_size = gz_path.stat().st_size
|
||||
if gz_size < orig_size:
|
||||
fpath.unlink()
|
||||
saved_bytes += (orig_size - gz_size)
|
||||
else:
|
||||
gz_path.unlink() # compression didn't help
|
||||
compressed += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to compress {fpath}: {e}")
|
||||
else:
|
||||
compressed += 1
|
||||
|
||||
return {"compressed": compressed, "saved_bytes": saved_bytes, "dry_run": dry_run}
|
||||
|
||||
|
||||
@router.post("/api/backups/auto", response_model=BackupsAutoResponse)
|
||||
async def api_backups_auto(
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Create backups for files modified since a given time. Body: {since_hours: 24}"""
|
||||
since_hours = body.get("since_hours", 24)
|
||||
cutoff = time.time() - (since_hours * 3600)
|
||||
backed_up = 0
|
||||
|
||||
for vault_name in index:
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
continue
|
||||
vd = get_vault_data(vault_name)
|
||||
if not vd:
|
||||
continue
|
||||
vault_root = Path(vd["path"])
|
||||
for fpath in vault_root.rglob("*"):
|
||||
if not fpath.is_file():
|
||||
continue
|
||||
if fpath.name.startswith('.'):
|
||||
continue
|
||||
if any(p.startswith('.') or p in {'.obsidian', '.trash', '.git', '.obsigate-backup', '__pycache__', 'node_modules'} for p in fpath.relative_to(vault_root).parts):
|
||||
continue
|
||||
mtime = fpath.stat().st_mtime
|
||||
if mtime < cutoff:
|
||||
continue
|
||||
try:
|
||||
rel = str(fpath.relative_to(vault_root)).replace("\\", "/")
|
||||
create_backup(fpath, vault_name, rel)
|
||||
backed_up += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Auto-backup failed for {rel}: {e}")
|
||||
|
||||
return {"backed_up": backed_up, "since_hours": since_hours}
|
||||
@@ -0,0 +1,828 @@
|
||||
"""API du coffre Secrets ObsiGate (Phase 1b, spec §7, §10, §11).
|
||||
|
||||
Monté dans ``backend/main.py`` à côté de ``config_router`` — JAMAIS dans
|
||||
``backend/routers/vaults.py`` : un coffre n'est pas un vault de notes, il
|
||||
ne passe ni par l'indexeur, ni par la recherche, ni par le graphe, et
|
||||
aucun outil MCP de secret n'existe.
|
||||
|
||||
Règles d'or, appliquées côté serveur à chaque requête :
|
||||
- Toutes les lectures renvoient des métadonnées seules (``has_value``,
|
||||
version, dates, octrois actifs) : jamais de valeur, jamais de texte
|
||||
chiffré, jamais de nonce.
|
||||
- Seul ``POST /api/coffres/distribuer`` rend une valeur, et uniquement à
|
||||
un consommateur authentifié par jeton (hash + jti, jamais la session
|
||||
utilisateur), avec un octroi actif et un contexte obligatoire.
|
||||
- Seule exception, voulue : ``POST /api/coffres/{slug}/secrets/{nom}/reveler``
|
||||
rend la valeur au gestionnaire du coffre authentifié par sa session
|
||||
(propriétaire, ou admin sur coffre d'instance — jamais un simple
|
||||
lecteur par octroi utilisateur), pour l'affichage et la copie manuelle
|
||||
dans le panneau détail. Chaque révélation est auditée
|
||||
(``revelation_manuelle``) et met à jour la dernière utilisation ; la
|
||||
valeur n'est jamais journalisée et le frontend l'efface dès qu'elle
|
||||
est masquée ou que le détail change.
|
||||
- Autorisations (spec §6) : le propriétaire seul sur son coffre
|
||||
personnel ; l'admin gère les coffres d'instance, les consommateurs et
|
||||
la KEK, jamais le coffre personnel d'autrui ; l'ACL « * » des vaults
|
||||
ne donne AUCUN droit sur un coffre (``check_vault_access`` n'est pas
|
||||
utilisé ici, volontairement).
|
||||
- Feature flag ``OBSIGATE_SECRETS_ENABLED`` (défaut désactivé) : tant
|
||||
qu'il n'est pas activé, toutes les routes répondent 503.
|
||||
- Phase 2 (issues #20) : migration ``api_keys.json`` -> coffre
|
||||
d'instance, en deux temps (import, puis purge confirmée), et
|
||||
enregistrement d'un consommateur avec ses octrois en un appel.
|
||||
- Phase 3 (issue #21) : ``/distribuer`` accepte le contexte Spark à
|
||||
plat ou imbriqué (``contexte: {type: "mission", id}``) ; les codes
|
||||
d'erreur restent ceux de la Phase 1b (voir
|
||||
``docs/COFFRE-contrat-distribution.md`` pour la correspondance avec
|
||||
l'état ``acces_requis`` côté Spark).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request
|
||||
|
||||
from backend.audit import get_recent_entries, log_coffre_event
|
||||
from backend.auth.middleware import require_admin, require_auth
|
||||
from backend.coffre_crypto import (
|
||||
CryptoIndisponibleError,
|
||||
KEKAbsenteError,
|
||||
generate_kek_b64,
|
||||
)
|
||||
from backend.coffre_migration import (
|
||||
COFFRE_INTERNE_SLUG,
|
||||
creer_consommateur_avec_octrois,
|
||||
importer_api_keys,
|
||||
lister_candidats,
|
||||
purger_api_keys,
|
||||
)
|
||||
from backend.coffre_store import (
|
||||
CoffreExisteError,
|
||||
IntrouvableError,
|
||||
SecretExisteError,
|
||||
SecretExpireError,
|
||||
ValidationError,
|
||||
compter_octrois_actifs,
|
||||
compter_secrets,
|
||||
create_coffre,
|
||||
create_consommateur,
|
||||
create_octroi,
|
||||
create_secret,
|
||||
delete_secret,
|
||||
distribuer_valeur,
|
||||
est_expire,
|
||||
get_coffre,
|
||||
get_coffre_by_id,
|
||||
get_coffre_or_404,
|
||||
get_octroi,
|
||||
get_secret,
|
||||
list_coffres,
|
||||
list_consommateurs,
|
||||
list_octrois,
|
||||
list_secrets,
|
||||
revoquer_consommateur,
|
||||
revoquer_octroi,
|
||||
rotate_dek,
|
||||
rotate_kek,
|
||||
rotate_secret,
|
||||
secret_metadata,
|
||||
secrets_accessibles_utilisateur,
|
||||
statut_octroi_consommateur,
|
||||
toucher_utilisation,
|
||||
verifier_jeton_consommateur,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.coffres")
|
||||
|
||||
router = APIRouter(tags=["Coffres Secrets"])
|
||||
|
||||
CODES_ERREUR = {
|
||||
"octroi_absent",
|
||||
"octroi_revoque",
|
||||
"octroi_expire",
|
||||
"secret_expire",
|
||||
"kek_absente",
|
||||
"contexte_manquant",
|
||||
"consommateur_invalide",
|
||||
"consommateur_revoque",
|
||||
"crypto_indisponible",
|
||||
"secrets_desactives",
|
||||
}
|
||||
|
||||
|
||||
def secrets_actives() -> bool:
|
||||
"""Feature flag : défaut désactivé, activation explicite par l'opérateur."""
|
||||
return os.environ.get("OBSIGATE_SECRETS_ENABLED", "false").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
)
|
||||
|
||||
|
||||
def _exiger_actives() -> None:
|
||||
if not secrets_actives():
|
||||
raise HTTPException(status_code=503, detail="secrets_desactives")
|
||||
|
||||
|
||||
def _username(user: dict) -> str:
|
||||
return str(user.get("username", ""))
|
||||
|
||||
|
||||
def _est_admin(user: dict) -> bool:
|
||||
return user.get("role") == "admin"
|
||||
|
||||
|
||||
def _peut_voir_coffre(user: dict, coffre: dict) -> bool:
|
||||
"""Voir les métadonnées d'un coffre."""
|
||||
if coffre["proprietaire"] == _username(user):
|
||||
return True
|
||||
if _est_admin(user) and coffre["perimetre"] == "instance":
|
||||
return True
|
||||
# Octroi utilisateur (coffre-wide ou par secret) : accès en lecture.
|
||||
accessibles = secrets_accessibles_utilisateur(_username(user), coffre)
|
||||
if accessibles is None:
|
||||
return True
|
||||
return len(accessibles) > 0
|
||||
|
||||
|
||||
def _peut_gerer_coffre(user: dict, coffre: dict) -> bool:
|
||||
"""Créer / roter / supprimer des secrets et gérer les octrois."""
|
||||
if coffre["proprietaire"] == _username(user):
|
||||
return True
|
||||
return _est_admin(user) and coffre["perimetre"] == "instance"
|
||||
|
||||
|
||||
def _exiger_voir(user: dict, coffre: dict) -> None:
|
||||
if not _peut_voir_coffre(user, coffre):
|
||||
raise HTTPException(status_code=403, detail="Accès refusé à ce coffre")
|
||||
|
||||
|
||||
def _exiger_gerer(user: dict, coffre: dict) -> None:
|
||||
if not _peut_gerer_coffre(user, coffre):
|
||||
raise HTTPException(status_code=403, detail="Accès refusé à ce coffre")
|
||||
|
||||
|
||||
def _coffre_or_404(slug: str) -> dict:
|
||||
try:
|
||||
return get_coffre_or_404(slug)
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
def _secret_or_404(coffre: dict, nom: str) -> dict:
|
||||
try:
|
||||
secret = get_secret(coffre["id"], nom)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if secret is None:
|
||||
raise HTTPException(status_code=404, detail=f"Secret introuvable : {nom}")
|
||||
return secret
|
||||
|
||||
|
||||
def _metadata_coffre(coffre: dict) -> dict:
|
||||
return {
|
||||
"slug": coffre["slug"],
|
||||
"nom": coffre["nom"],
|
||||
"description": coffre.get("description", ""),
|
||||
"proprietaire": coffre["proprietaire"],
|
||||
"perimetre": coffre["perimetre"],
|
||||
"statut": coffre["statut"],
|
||||
"kek_configuree": coffre.get("dek_chiffree") is not None,
|
||||
"nb_secrets": compter_secrets(coffre["id"]),
|
||||
"cree_le": coffre["cree_le"],
|
||||
}
|
||||
|
||||
|
||||
def _metadata_secret(coffre: dict, secret: dict) -> dict:
|
||||
return secret_metadata(secret, compter_octrois_actifs(secret["id"]))
|
||||
|
||||
|
||||
def _secrets_visibles(user: dict, coffre: dict) -> list:
|
||||
"""Secrets visibles par l'utilisateur (métadonnées seules)."""
|
||||
if coffre["proprietaire"] == _username(user) or (
|
||||
_est_admin(user) and coffre["perimetre"] == "instance"
|
||||
):
|
||||
return [_metadata_secret(coffre, s) for s in list_secrets(coffre["id"])]
|
||||
accessibles = secrets_accessibles_utilisateur(_username(user), coffre)
|
||||
if accessibles is None:
|
||||
return [_metadata_secret(coffre, s) for s in list_secrets(coffre["id"])]
|
||||
return [
|
||||
_metadata_secret(coffre, s)
|
||||
for s in list_secrets(coffre["id"])
|
||||
if s["id"] in accessibles
|
||||
]
|
||||
|
||||
|
||||
def _audit(user: str, action: str, **kwargs) -> None:
|
||||
try:
|
||||
log_coffre_event(user, action, **kwargs)
|
||||
except Exception as exc: # l'audit ne doit jamais casser l'opération
|
||||
logger.warning("Échec audit coffre (%s) : %s", action, exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coffres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/coffres")
|
||||
async def api_list_coffres(current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
visibles = [c for c in list_coffres() if _peut_voir_coffre(current_user, c)]
|
||||
return [_metadata_coffre(c) for c in visibles]
|
||||
|
||||
|
||||
@router.post("/api/coffres")
|
||||
async def api_create_coffre(body: dict = Body(...), current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
slug = body.get("slug", "")
|
||||
perimetre = body.get("perimetre", "personnel")
|
||||
if perimetre == "instance" and not _est_admin(current_user):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Seul un admin peut créer un coffre d'instance"
|
||||
)
|
||||
try:
|
||||
coffre = create_coffre(
|
||||
slug=slug,
|
||||
nom=body.get("nom", ""),
|
||||
proprietaire=_username(current_user),
|
||||
perimetre=perimetre,
|
||||
description=body.get("description", ""),
|
||||
)
|
||||
except CoffreExisteError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"coffre_cree",
|
||||
coffre=coffre["slug"],
|
||||
resultat="ok",
|
||||
)
|
||||
return _metadata_coffre(coffre)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Consommateurs (admin seulement, jeton rendu une seule fois)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/coffres/consommateurs")
|
||||
async def api_list_consommateurs(current_user=Depends(require_admin)):
|
||||
_exiger_actives()
|
||||
return list_consommateurs()
|
||||
|
||||
|
||||
@router.post("/api/coffres/consommateurs")
|
||||
async def api_create_consommateur(
|
||||
body: dict = Body(...), current_user=Depends(require_admin)
|
||||
):
|
||||
_exiger_actives()
|
||||
try:
|
||||
consommateur, jeton = create_consommateur(
|
||||
nom=body.get("nom", ""), cree_par=_username(current_user)
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"consommateur_cree",
|
||||
consommateur=consommateur["nom"],
|
||||
resultat="ok",
|
||||
)
|
||||
# Le jeton n'est rendu qu'ici, jamais stocké en clair, jamais relisté.
|
||||
return {**consommateur, "jeton": jeton}
|
||||
|
||||
|
||||
@router.post("/api/coffres/consommateurs/avec-octrois")
|
||||
async def api_create_consommateur_avec_octrois(
|
||||
body: dict = Body(...), current_user=Depends(require_admin)
|
||||
):
|
||||
"""Phase 4 : enregistrer une app = un consommateur + ses octrois.
|
||||
|
||||
Le coffre n'est jamais modifié par cette opération. Le jeton n'est
|
||||
rendu qu'ici, jamais stocké en clair, jamais relisté. Un octroi en
|
||||
erreur (coffre ou secret introuvable) est signalé dans ``erreurs``
|
||||
sans annuler le consommateur ni les autres octrois.
|
||||
"""
|
||||
_exiger_actives()
|
||||
try:
|
||||
consommateur, jeton, octrois, erreurs = creer_consommateur_avec_octrois(
|
||||
nom=body.get("nom", ""),
|
||||
octrois_demandes=body.get("octrois") or [],
|
||||
cree_par=_username(current_user),
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"consommateur_cree",
|
||||
consommateur=consommateur["nom"],
|
||||
resultat="ok",
|
||||
)
|
||||
for octroi in octrois:
|
||||
coffre_octroi = get_coffre_by_id(octroi["coffre_id"])
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"octroi_cree",
|
||||
coffre=coffre_octroi["slug"] if coffre_octroi else None,
|
||||
resultat="ok",
|
||||
)
|
||||
return {**consommateur, "jeton": jeton, "octrois": octrois, "erreurs": erreurs}
|
||||
|
||||
|
||||
@router.delete("/api/coffres/consommateurs/{consommateur_id}")
|
||||
async def api_revoke_consommateur(
|
||||
consommateur_id: str, current_user=Depends(require_admin)
|
||||
):
|
||||
_exiger_actives()
|
||||
try:
|
||||
consommateur = revoquer_consommateur(consommateur_id)
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"consommateur_revoque",
|
||||
consommateur=consommateur["nom"],
|
||||
resultat="ok",
|
||||
)
|
||||
return consommateur
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Migration Phase 2 (admin seulement) : api_keys.json -> coffre d'instance.
|
||||
# L'import ne supprime rien ; la purge exige une confirmation explicite
|
||||
# et ne supprime que les noms listés. Aucune réponse ne contient jamais
|
||||
# une valeur, seulement des noms et des comptes.
|
||||
# Ces routes sont déclarées AVANT /api/coffres/{slug} pour ne pas être
|
||||
# capturées par le paramètre slug.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/coffres/migration/api-keys")
|
||||
async def api_migration_apercu(current_user=Depends(require_admin)):
|
||||
_exiger_actives()
|
||||
return {"coffre": COFFRE_INTERNE_SLUG, "candidats": lister_candidats()}
|
||||
|
||||
|
||||
@router.post("/api/coffres/migration/api-keys/import")
|
||||
async def api_migration_import(current_user=Depends(require_admin)):
|
||||
_exiger_actives()
|
||||
try:
|
||||
resultat = importer_api_keys(cree_par=_username(current_user))
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except KEKAbsenteError:
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
raise HTTPException(status_code=503, detail="crypto_indisponible") from None
|
||||
for nom in resultat["migres"]:
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"migration_api_keys",
|
||||
coffre=COFFRE_INTERNE_SLUG,
|
||||
secret_ref=f"coffre:{COFFRE_INTERNE_SLUG}/{nom}",
|
||||
resultat="ok",
|
||||
)
|
||||
return resultat
|
||||
|
||||
|
||||
@router.post("/api/coffres/migration/api-keys/purger")
|
||||
async def api_migration_purger(
|
||||
body: dict = Body(...), current_user=Depends(require_admin)
|
||||
):
|
||||
_exiger_actives()
|
||||
if body.get("confirmer") is not True:
|
||||
raise HTTPException(status_code=400, detail="confirmation_requise")
|
||||
try:
|
||||
resultat = purger_api_keys(
|
||||
noms=body.get("noms") or [], confirmer=True
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
for nom in resultat["supprimes"]:
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"migration_api_keys_purge",
|
||||
coffre=COFFRE_INTERNE_SLUG,
|
||||
secret_ref=f"coffre:{COFFRE_INTERNE_SLUG}/{nom}",
|
||||
resultat="ok",
|
||||
)
|
||||
return resultat
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Distribution (serveur-à-serveur, hors session utilisateur, hors LLM)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/api/coffres/distribuer")
|
||||
async def api_distribuer(request: Request, body: dict = Body(...)):
|
||||
_exiger_actives()
|
||||
reference = (body.get("reference") or "").strip()
|
||||
# Phase 3 (Spark) : le contexte peut arriver à plat
|
||||
# (contexte_type / contexte_id) ou imbriqué (contexte: {type, id},
|
||||
# forme envoyée par le client Spark). Les deux formes sont acceptées.
|
||||
contexte_type = str(body.get("contexte_type") or "").strip()
|
||||
contexte_id_raw = body.get("contexte_id")
|
||||
contexte_id = "" if contexte_id_raw is None else str(contexte_id_raw).strip()
|
||||
if not contexte_type or not contexte_id:
|
||||
contexte = body.get("contexte")
|
||||
if isinstance(contexte, dict):
|
||||
contexte_type = contexte_type or str(contexte.get("type") or "").strip()
|
||||
contexte_id_raw = contexte.get("id")
|
||||
contexte_id = contexte_id or (
|
||||
"" if contexte_id_raw is None else str(contexte_id_raw).strip()
|
||||
)
|
||||
if not contexte_type or not contexte_id:
|
||||
raise HTTPException(status_code=400, detail="contexte_manquant")
|
||||
if not reference.startswith("coffre:") or "/" not in reference[7:]:
|
||||
raise HTTPException(status_code=400, detail="reference_invalide")
|
||||
slug, nom = reference[7:].split("/", 1)
|
||||
# Authentification consommateur : Bearer, hash + statut, jamais un JWT
|
||||
# de session utilisateur.
|
||||
auth = request.headers.get("Authorization", "")
|
||||
jeton = auth[7:] if auth.startswith("Bearer ") else ""
|
||||
consommateur = verifier_jeton_consommateur(jeton)
|
||||
if consommateur is None:
|
||||
_audit("consommateur", "distribution_refusee", secret_ref=reference,
|
||||
resultat="consommateur_invalide", contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=401, detail="consommateur_invalide")
|
||||
if consommateur["statut"] != "actif":
|
||||
_audit("consommateur", "distribution_refusee", secret_ref=reference,
|
||||
consommateur=consommateur["nom"],
|
||||
resultat="consommateur_revoque", contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=403, detail="consommateur_revoque")
|
||||
try:
|
||||
coffre = get_coffre(slug)
|
||||
secret = get_secret(coffre["id"], nom) if coffre else None
|
||||
except ValidationError:
|
||||
raise HTTPException(status_code=400, detail="reference_invalide") from None
|
||||
if coffre is None or secret is None or coffre["statut"] != "actif":
|
||||
# Anti-énumération : un secret inexistant se comporte comme un
|
||||
# octroi absent, il ne révèle pas l'existence du coffre/secret.
|
||||
_audit("consommateur", "distribution_refusee", secret_ref=reference,
|
||||
consommateur=consommateur["nom"],
|
||||
resultat="octroi_absent", contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=403, detail="octroi_absent")
|
||||
statut, _octroi = statut_octroi_consommateur(
|
||||
consommateur["id"], coffre["id"], secret["id"]
|
||||
)
|
||||
if statut != "ok":
|
||||
code = {
|
||||
"revoque": "octroi_revoque",
|
||||
"expire": "octroi_expire",
|
||||
"absent": "octroi_absent",
|
||||
}[statut]
|
||||
_audit("consommateur", "distribution_refusee", coffre=coffre["slug"],
|
||||
secret_ref=reference, consommateur=consommateur["nom"],
|
||||
resultat=code, contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=403, detail=code)
|
||||
try:
|
||||
valeur = distribuer_valeur(coffre, secret)
|
||||
except SecretExpireError:
|
||||
_audit("consommateur", "distribution_refusee", coffre=coffre["slug"],
|
||||
secret_ref=reference, consommateur=consommateur["nom"],
|
||||
resultat="secret_expire", contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=410, detail="secret_expire") from None
|
||||
except KEKAbsenteError:
|
||||
_audit("consommateur", "distribution_refusee", coffre=coffre["slug"],
|
||||
secret_ref=reference, consommateur=consommateur["nom"],
|
||||
resultat="kek_absente", contexte_id=contexte_id)
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
raise HTTPException(status_code=503, detail="crypto_indisponible") from None
|
||||
toucher_utilisation(secret["id"])
|
||||
_audit("consommateur", "distribution_ok", coffre=coffre["slug"],
|
||||
secret_ref=reference, consommateur=consommateur["nom"],
|
||||
resultat="ok", contexte_id=contexte_id)
|
||||
return {
|
||||
"reference": reference,
|
||||
"valeur": valeur,
|
||||
"version": secret["version_courante"],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/coffres/distribuables")
|
||||
async def api_distribuables(request: Request):
|
||||
"""Secrets distribuables pour un consommateur (issue #43).
|
||||
|
||||
Authentification par jeton consommateur (la même que
|
||||
/distribuer), jamais par session utilisateur : c'est le
|
||||
serveur agent qui appelle, pour alimenter la liste déroulante
|
||||
de ses fournisseurs LLM. Retour = métadonnées seules des
|
||||
secrets pour lesquels ce consommateur a un octroi ACTIF
|
||||
(coffre actif, secret non expiré, octroi ni révoqué ni
|
||||
expiré, y compris les octrois coffre-wide). Un consommateur
|
||||
sans octroi reçoit une liste vide — jamais d'erreur
|
||||
révélatrice, jamais de valeur.
|
||||
"""
|
||||
_exiger_actives()
|
||||
auth = request.headers.get("Authorization", "")
|
||||
jeton = auth[7:] if auth.startswith("Bearer ") else ""
|
||||
consommateur = verifier_jeton_consommateur(jeton)
|
||||
if consommateur is None:
|
||||
raise HTTPException(status_code=401,
|
||||
detail="consommateur_invalide")
|
||||
if consommateur["statut"] != "actif":
|
||||
raise HTTPException(status_code=403,
|
||||
detail="consommateur_revoque")
|
||||
sortie = []
|
||||
for coffre in list_coffres():
|
||||
if coffre["statut"] != "actif":
|
||||
continue
|
||||
for secret in list_secrets(coffre["id"]):
|
||||
if est_expire(secret.get("expiration")):
|
||||
continue
|
||||
statut, _octroi = statut_octroi_consommateur(
|
||||
consommateur["id"], coffre["id"], secret["id"])
|
||||
if statut != "ok":
|
||||
continue
|
||||
sortie.append({
|
||||
"reference": f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
"coffre": coffre["slug"],
|
||||
"nom": secret["nom"],
|
||||
"version": secret["version_courante"],
|
||||
})
|
||||
return sortie
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Un coffre, ses secrets, ses octrois, son journal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/coffres/{slug}")
|
||||
async def api_get_coffre(slug: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_voir(current_user, coffre)
|
||||
return _metadata_coffre(coffre)
|
||||
|
||||
|
||||
@router.get("/api/coffres/{slug}/secrets")
|
||||
async def api_list_secrets(slug: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_voir(current_user, coffre)
|
||||
return _secrets_visibles(current_user, coffre)
|
||||
|
||||
|
||||
@router.post("/api/coffres/{slug}/secrets")
|
||||
async def api_create_secret(
|
||||
slug: str, body: dict = Body(...), current_user=Depends(require_auth)
|
||||
):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
try:
|
||||
secret = create_secret(
|
||||
coffre=coffre,
|
||||
nom=body.get("nom", ""),
|
||||
type_secret=body.get("type", "autre"),
|
||||
valeur=body.get("valeur", ""),
|
||||
service=body.get("service", ""),
|
||||
expiration=body.get("expiration"),
|
||||
note=body.get("note", ""),
|
||||
)
|
||||
except SecretExisteError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except KEKAbsenteError:
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
raise HTTPException(status_code=503, detail="crypto_indisponible") from None
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"secret_cree",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="ok",
|
||||
)
|
||||
return _metadata_secret(coffre, secret)
|
||||
|
||||
|
||||
@router.post("/api/coffres/{slug}/secrets/{nom}/rotation")
|
||||
async def api_rotate_secret(
|
||||
slug: str, nom: str, body: dict = Body(...), current_user=Depends(require_auth)
|
||||
):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
try:
|
||||
secret = rotate_secret(coffre, nom, body.get("valeur", ""))
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except KEKAbsenteError:
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
raise HTTPException(status_code=503, detail="crypto_indisponible") from None
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"secret_rotation",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="ok",
|
||||
)
|
||||
return _metadata_secret(coffre, secret)
|
||||
|
||||
|
||||
@router.post("/api/coffres/{slug}/secrets/{nom}/reveler")
|
||||
async def api_reveler_secret(
|
||||
slug: str, nom: str, current_user=Depends(require_auth)
|
||||
):
|
||||
"""Révélation manuelle pour le panneau détail (afficher / copier).
|
||||
|
||||
Session utilisateur + gestion du coffre exigées : un lecteur par
|
||||
octroi utilisateur voit les métadonnées mais ne révèle jamais la
|
||||
valeur. Chaque appel est audité et touche la dernière utilisation.
|
||||
"""
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
secret = _secret_or_404(coffre, nom)
|
||||
try:
|
||||
valeur = distribuer_valeur(coffre, secret)
|
||||
except SecretExpireError:
|
||||
_audit(_username(current_user), "revelation_refusee",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="secret_expire")
|
||||
raise HTTPException(status_code=410, detail="secret_expire") from None
|
||||
except KEKAbsenteError:
|
||||
_audit(_username(current_user), "revelation_refusee",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="kek_absente")
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
_audit(_username(current_user), "revelation_refusee",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="crypto_indisponible")
|
||||
raise HTTPException(status_code=503,
|
||||
detail="crypto_indisponible") from None
|
||||
toucher_utilisation(secret["id"])
|
||||
_audit(_username(current_user), "revelation_manuelle",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="ok")
|
||||
return {"reference": f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
"valeur": valeur, "version": secret["version_courante"]}
|
||||
|
||||
|
||||
@router.delete("/api/coffres/{slug}/secrets/{nom}")
|
||||
async def api_delete_secret(
|
||||
slug: str, nom: str, current_user=Depends(require_auth)
|
||||
):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
try:
|
||||
secret, _octrois = delete_secret(coffre, nom)
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"secret_supprime",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=f"coffre:{coffre['slug']}/{secret['nom']}",
|
||||
resultat="ok",
|
||||
)
|
||||
return {"status": "deleted", "nom": secret["nom"]}
|
||||
|
||||
|
||||
@router.get("/api/coffres/{slug}/octrois")
|
||||
async def api_list_octrois(slug: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_voir(current_user, coffre)
|
||||
return list_octrois(coffre["id"])
|
||||
|
||||
|
||||
@router.post("/api/coffres/{slug}/octrois")
|
||||
async def api_create_octroi(
|
||||
slug: str, body: dict = Body(...), current_user=Depends(require_auth)
|
||||
):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
secret_id = None
|
||||
if body.get("secret_nom"):
|
||||
secret = _secret_or_404(coffre, body["secret_nom"])
|
||||
secret_id = secret["id"]
|
||||
try:
|
||||
octroi = create_octroi(
|
||||
coffre=coffre,
|
||||
cible_type=body.get("cible_type", ""),
|
||||
cible_id=body.get("cible_id", ""),
|
||||
secret_id=secret_id,
|
||||
cree_par=_username(current_user),
|
||||
expiration=body.get("expiration"),
|
||||
)
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"octroi_cree",
|
||||
coffre=coffre["slug"],
|
||||
secret_ref=(
|
||||
f"coffre:{coffre['slug']}/{body.get('secret_nom', '*')}"
|
||||
),
|
||||
resultat="ok",
|
||||
)
|
||||
return octroi
|
||||
|
||||
|
||||
@router.delete("/api/coffres/octrois/{octroi_id}")
|
||||
async def api_revoke_octroi(octroi_id: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
# Le gestionnaire du coffre concerné (ou l'admin d'instance) seul peut
|
||||
# révoquer : on résout le coffre depuis l'octroi avant toute action.
|
||||
octroi = get_octroi(octroi_id)
|
||||
if octroi is None:
|
||||
raise HTTPException(status_code=404, detail="Octroi introuvable")
|
||||
coffre = get_coffre_by_id(octroi["coffre_id"])
|
||||
if coffre is None:
|
||||
raise HTTPException(status_code=404, detail="Coffre introuvable")
|
||||
_exiger_gerer(current_user, coffre)
|
||||
try:
|
||||
octroi = revoquer_octroi(octroi_id)
|
||||
except IntrouvableError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
_audit(
|
||||
_username(current_user),
|
||||
"octroi_revoque",
|
||||
coffre=coffre["slug"],
|
||||
resultat="ok",
|
||||
)
|
||||
return octroi
|
||||
|
||||
|
||||
@router.get("/api/coffres/{slug}/journal")
|
||||
async def api_journal(slug: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_voir(current_user, coffre)
|
||||
# Défense en profondeur : même si une entrée contenait un jour un champ
|
||||
# sensible, il ne sort jamais par cette route.
|
||||
nettoyees = []
|
||||
for entry in get_recent_entries(limit=100):
|
||||
if entry.get("coffre") != slug:
|
||||
continue
|
||||
nettoyees.append(
|
||||
{
|
||||
k: v
|
||||
for k, v in entry.items()
|
||||
if k not in ("valeur", "ciphertext", "nonce", "jeton", "dek_chiffree")
|
||||
}
|
||||
)
|
||||
return nettoyees
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Administration KEK / rotations (admin seulement)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/api/coffres/admin/generer-kek")
|
||||
async def api_generer_kek(current_user=Depends(require_admin)):
|
||||
_exiger_actives()
|
||||
# La KEK générée n'est rendue qu'une fois, ici, et jamais stockée par
|
||||
# ObsiGate : l'opérateur doit la sauvegarder hors dépôt immédiatement.
|
||||
return {"kek": generate_kek_b64()}
|
||||
|
||||
|
||||
@router.post("/api/coffres/{slug}/rotation-dek")
|
||||
async def api_rotate_dek(slug: str, current_user=Depends(require_auth)):
|
||||
_exiger_actives()
|
||||
coffre = _coffre_or_404(slug)
|
||||
_exiger_gerer(current_user, coffre)
|
||||
try:
|
||||
coffre = rotate_dek(slug)
|
||||
except KEKAbsenteError:
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except CryptoIndisponibleError:
|
||||
raise HTTPException(status_code=503, detail="crypto_indisponible") from None
|
||||
_audit(_username(current_user), "dek_rotation", coffre=coffre["slug"],
|
||||
resultat="ok")
|
||||
return _metadata_coffre(coffre)
|
||||
|
||||
|
||||
@router.post("/api/coffres/admin/rotation-kek")
|
||||
async def api_rotate_kek(
|
||||
body: dict = Body(...), current_user=Depends(require_admin)
|
||||
):
|
||||
_exiger_actives()
|
||||
try:
|
||||
count = rotate_kek(body.get("nouvelle_kek", ""))
|
||||
except KEKAbsenteError:
|
||||
raise HTTPException(status_code=503, detail="kek_absente") from None
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
_audit(_username(current_user), "kek_rotation", resultat="ok")
|
||||
return {"status": "ok", "coffres_rechiffres": count}
|
||||
@@ -0,0 +1,424 @@
|
||||
# backend/routers/comptes_machine.py
|
||||
# HUB-13 (fusion #127) — API d'administration des comptes, machine-à-machine.
|
||||
#
|
||||
# Le Hub (interface d'administration des utilisateurs) administre les
|
||||
# comptes dont ObsiGate reste le magasin (décisions D1(a)/D2(a) du plan
|
||||
# Hub v2) via ces routes, en lecture et en écriture : lister, lire le
|
||||
# profil étendu, créer, modifier, désactiver/réactiver, supprimer,
|
||||
# réinitialiser le mot de passe et le MFA.
|
||||
#
|
||||
# HUB-22 (#227, vague C du plan Hub v2.1) ajoute, sous la même garde et
|
||||
# avec la même journalisation, les routes « sessions » : lister les
|
||||
# sessions draco actives d'un compte, en révoquer une (par son `jti`,
|
||||
# jamais celle d'un autre compte) ou toutes (déconnexion globale, sans
|
||||
# désactiver le compte), via le registre des émissions de HUB-13.
|
||||
#
|
||||
# Emplacement : ``/api/admin/machine/comptes`` — dans l'espace
|
||||
# d'administration existant (``/api/admin/…``), avec un segment
|
||||
# ``machine`` explicite qui le distingue des routes d'administration par
|
||||
# session (``/api/auth/admin/users``, garde ``require_admin``). Aucune
|
||||
# session utilisateur — même administratrice — n'ouvre ces routes :
|
||||
# seule la clé machine dédiée les protège.
|
||||
#
|
||||
# Authentification machine (même famille que ``GET /api/sso/revocations``) :
|
||||
# DRACO_ADMIN_MACHINE_KEY clé dédiée à cette API (en-tête
|
||||
# ``X-Draco-Machine-Key``). Distincte de
|
||||
# ``DRACO_SSO_MACHINE_KEY`` : le périmètre
|
||||
# (écriture de comptes) est plus sensible que
|
||||
# la seule lecture des révocations (plan §3).
|
||||
# Sans elle, l'API est inerte (404, comme la
|
||||
# liste de révocations non configurée).
|
||||
# X-Draco-Actor identité de service appelante, facultative
|
||||
# (défaut ``service-machine``) — consignée
|
||||
# comme acteur dans le journal d'audit.
|
||||
#
|
||||
# Chaque action est journalisée côté ObsiGate dans le journal d'audit
|
||||
# existant (``data/audit.log``, voir backend.audit.log_compte_admin).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, validator
|
||||
|
||||
from backend import draco_session
|
||||
from backend.audit import log_compte_admin
|
||||
from backend.auth.password import validate_password_strength
|
||||
from backend.auth.user_store import (
|
||||
create_user,
|
||||
delete_user,
|
||||
get_all_users,
|
||||
get_user,
|
||||
update_user,
|
||||
)
|
||||
from backend.services.net import get_client_ip
|
||||
|
||||
logger = logging.getLogger("obsigate.comptes_machine")
|
||||
|
||||
router = APIRouter(prefix="/api/admin/machine/comptes", tags=["admin"])
|
||||
|
||||
#: Variable d'environnement portant la clé machine dédiée à cette API.
|
||||
MACHINE_KEY_ENV = "DRACO_ADMIN_MACHINE_KEY"
|
||||
#: En-tête de la clé machine (même nom que pour /api/sso/revocations).
|
||||
MACHINE_KEY_HEADER = "X-Draco-Machine-Key"
|
||||
#: En-tête facultatif : identité de service appelante (journal d'audit).
|
||||
ACTOR_HEADER = "X-Draco-Actor"
|
||||
#: Acteur consigné quand l'appelant ne se nomme pas.
|
||||
DEFAULT_ACTOR = "service-machine"
|
||||
|
||||
|
||||
# ── Garde machine ───────────────────────────────────────────────────────
|
||||
|
||||
def require_machine_admin(request: Request) -> str:
|
||||
"""Exige la clé machine dédiée ; retourne l'identité de service.
|
||||
|
||||
Jamais de repli sur une session utilisateur : sans en-tête valide,
|
||||
la réponse est 401 (ou 404 si l'API n'est pas configurée, patron de
|
||||
``GET /api/sso/revocations``).
|
||||
"""
|
||||
machine_key = os.environ.get(MACHINE_KEY_ENV, "")
|
||||
if not machine_key:
|
||||
raise HTTPException(404, "Non configuré")
|
||||
provided = request.headers.get(MACHINE_KEY_HEADER, "")
|
||||
if not provided or not hmac.compare_digest(provided, machine_key):
|
||||
raise HTTPException(401, "Clé machine invalide")
|
||||
actor = (request.headers.get(ACTOR_HEADER, "") or "").strip()
|
||||
return actor[:64] if actor else DEFAULT_ACTOR
|
||||
|
||||
|
||||
# ── Profil étendu (liste blanche — jamais de secret) ────────────────────
|
||||
|
||||
def _profil(user: dict) -> dict:
|
||||
"""Profil étendu d'un compte, sûr à exposer à un service.
|
||||
|
||||
Liste blanche stricte : ni ``password_hash``, ni secret TOTP
|
||||
(``mfa_secret``/``mfa_secret_pending``), ni codes de récupération,
|
||||
ni identifiants WebAuthn ne sortent — seul l'état MFA (oui/non)
|
||||
est exposé. L'avatar suit le modèle existant : la valeur stockée
|
||||
(data-URL ou ``None``, comme ``GET /api/auth/me``) plus le booléen
|
||||
de présence ``avatar_present`` pour les listes légères.
|
||||
"""
|
||||
return {
|
||||
"id": user.get("id"),
|
||||
"username": user.get("username"),
|
||||
"display_name": user.get("display_name"),
|
||||
"language": user.get("language", "fr"),
|
||||
"avatar": user.get("avatar"),
|
||||
"avatar_present": bool(user.get("avatar")),
|
||||
"role": user.get("role"),
|
||||
"active": bool(user.get("active")),
|
||||
"vaults": user.get("vaults", []),
|
||||
"mfa_enabled": bool(user.get("mfa_enabled")),
|
||||
"mfa_method": user.get("mfa_method"),
|
||||
"must_change_password": bool(user.get("must_change_password")),
|
||||
"created_at": user.get("created_at"),
|
||||
"last_login": user.get("last_login"),
|
||||
}
|
||||
|
||||
|
||||
def _compte_ou_404(username: str) -> dict:
|
||||
user = get_user(username)
|
||||
if user is None:
|
||||
raise HTTPException(404, "Compte introuvable")
|
||||
return user
|
||||
|
||||
|
||||
def _identifiant_sessions(user: dict) -> str:
|
||||
"""Clé du registre des émissions draco pour un compte.
|
||||
|
||||
C'est le ``sub`` des jetons draco : l'identifiant du compte
|
||||
(``id``), à défaut son nom d'utilisateur — exactement la clé
|
||||
qu'utilise ``_revoquer_sessions`` depuis HUB-13.
|
||||
"""
|
||||
return str(user.get("id") or user.get("username") or "")
|
||||
|
||||
|
||||
def _revoquer_sessions(user: dict) -> int:
|
||||
"""Révoque les sessions d'un compte ; retourne le nombre de jti draco.
|
||||
|
||||
Les ``jti`` du cookie partagé émis pour ce compte sont révoqués via
|
||||
le mécanisme existant (``draco_session.revoke``, exposé aux autres
|
||||
applications par ``GET /api/sso/revocations``). Les jetons natifs
|
||||
sont invalidés par le garde ``password_changed_at`` (BUG-028),
|
||||
avancé par l'appelant (désactivation, réinitialisation du mot de
|
||||
passe) ou par le drapeau ``active`` tant que le compte est inactif.
|
||||
"""
|
||||
return draco_session.revoke_all_for_user(_identifiant_sessions(user))
|
||||
|
||||
|
||||
def _audit(request: Request, acteur: str, action: str, cible: str,
|
||||
details: dict | None = None) -> None:
|
||||
log_compte_admin(acteur, action, cible,
|
||||
ip=get_client_ip(request), details=details)
|
||||
|
||||
|
||||
# ── Modèles de requête ──────────────────────────────────────────────────
|
||||
|
||||
class CreerCompteRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
display_name: str | None = None
|
||||
role: str = "user"
|
||||
vaults: list[str] = []
|
||||
|
||||
@validator("password")
|
||||
def password_valid(cls, v):
|
||||
return validate_password_strength(v)
|
||||
|
||||
@validator("username")
|
||||
def username_valid(cls, v):
|
||||
if not re.match(r"^[a-zA-Z0-9_-]{2,32}$", v):
|
||||
raise ValueError("2-32 caractères alphanumériques, _ ou -")
|
||||
return v.lower()
|
||||
|
||||
@validator("role")
|
||||
def role_valid(cls, v):
|
||||
if v not in ("admin", "user"):
|
||||
raise ValueError("Rôle invalide")
|
||||
return v
|
||||
|
||||
|
||||
class ModifierCompteRequest(BaseModel):
|
||||
display_name: str | None = None
|
||||
role: str | None = None
|
||||
language: str | None = None
|
||||
vaults: list[str] | None = None
|
||||
|
||||
@validator("role")
|
||||
def role_valid(cls, v):
|
||||
if v is not None and v not in ("admin", "user"):
|
||||
raise ValueError("Rôle invalide")
|
||||
return v
|
||||
|
||||
@validator("language")
|
||||
def language_valid(cls, v):
|
||||
if v is not None and v not in ("fr", "en"):
|
||||
raise ValueError("Langue invalide (fr | en)")
|
||||
return v
|
||||
|
||||
|
||||
class ReinitialiserMotDePasseRequest(BaseModel):
|
||||
password: str
|
||||
|
||||
@validator("password")
|
||||
def password_valid(cls, v):
|
||||
return validate_password_strength(v)
|
||||
|
||||
|
||||
# ── Lecture ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("")
|
||||
async def lister_comptes(acteur: str = Depends(require_machine_admin)):
|
||||
"""Liste tous les comptes (profils étendus, sans aucun secret)."""
|
||||
comptes = [_profil(u) for u in get_all_users()]
|
||||
return {"comptes": comptes, "total": len(comptes)}
|
||||
|
||||
|
||||
@router.get("/{username}")
|
||||
async def lire_compte(username: str,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Lit un compte et son profil étendu."""
|
||||
return _profil(_compte_ou_404(username))
|
||||
|
||||
|
||||
# ── Écriture ────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def creer_compte(req: CreerCompteRequest, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Crée un compte. Le mot de passe initial est temporaire :
|
||||
``must_change_password`` est posé (changement forcé signalé à la
|
||||
première connexion, comme après une réinitialisation)."""
|
||||
if get_user(req.username) is not None:
|
||||
raise HTTPException(409, "Identifiant déjà utilisé")
|
||||
try:
|
||||
user = create_user(
|
||||
req.username, req.password, req.role, req.vaults,
|
||||
req.display_name, must_change_password=True,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
# #194 : dossier personnel — même provisionnement que la création
|
||||
# par session administratrice (best effort).
|
||||
from backend.user_home import ensure_user_home
|
||||
|
||||
await ensure_user_home(req.username)
|
||||
fresh = get_user(req.username)
|
||||
if fresh:
|
||||
user = fresh
|
||||
_audit(request, acteur, "compte_cree", req.username,
|
||||
{"role": req.role})
|
||||
return _profil(user)
|
||||
|
||||
|
||||
@router.patch("/{username}")
|
||||
async def modifier_compte(username: str, req: ModifierCompteRequest,
|
||||
request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Modifie un compte (nom affiché, rôle global, langue, vaults)."""
|
||||
_compte_ou_404(username)
|
||||
updates = req.dict(exclude_none=True)
|
||||
if not updates:
|
||||
raise HTTPException(400, "Aucun champ à modifier")
|
||||
updated = update_user(username, updates)
|
||||
_audit(request, acteur, "compte_modifie", username,
|
||||
{"champs": sorted(updates.keys())})
|
||||
return _profil(updated)
|
||||
|
||||
|
||||
@router.post("/{username}/desactiver")
|
||||
async def desactiver_compte(username: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Désactive un compte : plus aucun nouveau jeton n'est émis
|
||||
(connexion refusée) et les ``jti`` draco actifs sont révoqués."""
|
||||
user = _compte_ou_404(username)
|
||||
updated = update_user(username, {
|
||||
"active": False,
|
||||
# BUG-028 : avancer le repère invalide aussi les jetons natifs
|
||||
# déjà émis — ils ne ressuscitent pas à la réactivation.
|
||||
"password_changed_at": time.time(),
|
||||
})
|
||||
nb_jti = _revoquer_sessions(user)
|
||||
_audit(request, acteur, "compte_desactive", username,
|
||||
{"jti_revoques": nb_jti})
|
||||
return _profil(updated)
|
||||
|
||||
|
||||
@router.post("/{username}/reactiver")
|
||||
async def reactiver_compte(username: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Réactive un compte désactivé (les anciennes sessions restent
|
||||
révoquées : une nouvelle connexion est nécessaire)."""
|
||||
_compte_ou_404(username)
|
||||
updated = update_user(username, {"active": True})
|
||||
_audit(request, acteur, "compte_reactive", username)
|
||||
return _profil(updated)
|
||||
|
||||
|
||||
@router.delete("/{username}")
|
||||
async def supprimer_compte(username: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Supprime un compte et révoque ses ``jti`` draco actifs."""
|
||||
user = _compte_ou_404(username)
|
||||
nb_jti = _revoquer_sessions(user)
|
||||
try:
|
||||
delete_user(username)
|
||||
except ValueError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
# #194 : fermer le vault du dossier personnel (dossier conservé),
|
||||
# comme la suppression par session administratrice.
|
||||
from backend.user_home import release_user_home
|
||||
|
||||
await release_user_home(username)
|
||||
_audit(request, acteur, "compte_supprime", username,
|
||||
{"jti_revoques": nb_jti})
|
||||
return {"message": f"Compte '{username}' supprimé"}
|
||||
|
||||
|
||||
@router.post("/{username}/reinitialiser-mot-de-passe")
|
||||
async def reinitialiser_mot_de_passe(
|
||||
username: str, req: ReinitialiserMotDePasseRequest,
|
||||
request: Request, acteur: str = Depends(require_machine_admin)):
|
||||
"""Pose un nouveau mot de passe temporaire : changement forcé à la
|
||||
première connexion (``must_change_password``, signalé par la
|
||||
connexion et ``GET /api/auth/me``, effacé au changement effectif).
|
||||
Les sessions existantes sont révoquées (repère BUG-028 + ``jti``
|
||||
draco). Le mot de passe n'est jamais journalisé."""
|
||||
user = _compte_ou_404(username)
|
||||
updated = update_user(username, {
|
||||
"password": req.password,
|
||||
"must_change_password": True,
|
||||
})
|
||||
nb_jti = _revoquer_sessions(user)
|
||||
_audit(request, acteur, "mot_de_passe_reinitialise", username,
|
||||
{"jti_revoques": nb_jti})
|
||||
return _profil(updated)
|
||||
|
||||
|
||||
@router.post("/{username}/reinitialiser-mfa")
|
||||
async def reinitialiser_mfa(username: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Efface l'enrôlement TOTP d'un compte (utilisateur verrouillé
|
||||
hors de son second facteur) : secret, secret en attente et codes
|
||||
de récupération sont supprimés. Les clés WebAuthn éventuelles sont
|
||||
conservées et gardent alors le MFA actif. Le secret n'est jamais
|
||||
exposé — ni avant, ni après."""
|
||||
user = _compte_ou_404(username)
|
||||
webauthn_restantes = len(user.get("webauthn_credentials") or [])
|
||||
updates: dict = {
|
||||
"mfa_secret": None,
|
||||
"mfa_secret_pending": None,
|
||||
"mfa_recovery_codes": [],
|
||||
}
|
||||
if webauthn_restantes:
|
||||
updates["mfa_method"] = "webauthn"
|
||||
else:
|
||||
updates["mfa_enabled"] = False
|
||||
updates["mfa_method"] = None
|
||||
updated = update_user(username, updates)
|
||||
_audit(request, acteur, "mfa_reinitialise", username,
|
||||
{"webauthn_conserves": webauthn_restantes})
|
||||
return _profil(updated)
|
||||
|
||||
|
||||
# ── Sessions draco (HUB-22) ─────────────────────────────────────────────
|
||||
|
||||
@router.get("/{username}/sessions")
|
||||
async def lister_sessions(username: str,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Liste les sessions draco actives d'un compte.
|
||||
|
||||
Une session par ``jti`` émis et non expiré (registre des émissions
|
||||
de HUB-13), triées par expiration croissante ; ``revoquee`` signale
|
||||
un ``jti`` déjà présent dans la liste des révocations et
|
||||
``expire_le`` est l'échéance ``exp`` en ISO 8601 UTC.
|
||||
"""
|
||||
user = _compte_ou_404(username)
|
||||
sessions = [{
|
||||
"jti": s["jti"],
|
||||
"exp": s["exp"],
|
||||
"expire_le": datetime.fromtimestamp(s["exp"], tz=UTC).isoformat(),
|
||||
"revoquee": s["revoquee"],
|
||||
} for s in draco_session.issued_sessions_for_user(
|
||||
_identifiant_sessions(user))]
|
||||
return {"username": username, "sessions": sessions}
|
||||
|
||||
|
||||
@router.post("/{username}/sessions/{jti}/revoquer")
|
||||
async def revoquer_session(username: str, jti: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Révoque une seule session draco d'un compte, par son ``jti``.
|
||||
|
||||
404 si le ``jti`` ne figure pas dans l'ensemble actif de ce
|
||||
compte : une session d'un autre utilisateur n'est jamais révoquée
|
||||
par cette route (aucune révocation cross-utilisateur).
|
||||
"""
|
||||
user = _compte_ou_404(username)
|
||||
if not draco_session.revoke_one_for_user(
|
||||
_identifiant_sessions(user), jti):
|
||||
raise HTTPException(404, "Session introuvable")
|
||||
_audit(request, acteur, "session_revoquee", username, {"jti": jti})
|
||||
return {"jti": jti, "revoquee": True}
|
||||
|
||||
|
||||
@router.post("/{username}/sessions/revoquer-toutes")
|
||||
async def revoquer_toutes_sessions(
|
||||
username: str, request: Request,
|
||||
acteur: str = Depends(require_machine_admin)):
|
||||
"""Révoque toutes les sessions draco actives d'un compte.
|
||||
|
||||
C'est la « déconnexion globale » : contrairement à la
|
||||
désactivation, le compte reste actif — seule l'émission d'une
|
||||
nouvelle session (nouvelle connexion) rouvre l'accès.
|
||||
"""
|
||||
user = _compte_ou_404(username)
|
||||
nb_jti = _revoquer_sessions(user)
|
||||
_audit(request, acteur, "sessions_revoquees", username,
|
||||
{"jti_revoques": nb_jti})
|
||||
return {"revoquees": nb_jti}
|
||||
@@ -0,0 +1,530 @@
|
||||
"""Configuration, AI keys, diagnostics & dashboard endpoints (ROADMAP #85, tranche 7).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/config*``, ``/api/diagnostics``,
|
||||
``/api/dashboard``), mêmes modèles de réponse, mêmes dépendances
|
||||
d'authentification.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_load_config`` / ``_save_config`` / ``_DEFAULT_CONFIG`` /
|
||||
``_CONFIG_PATH`` / ``_BASE_DIR`` ont déménagé ici : ``main`` les
|
||||
réimporte pour son lifespan (pas de cycle : ce module ne dépend pas de
|
||||
``main``).
|
||||
- ``AI_KEYS_FILE`` / ``_write_ai_keys`` / ``_FALLBACK_MODELS`` ont déménagé
|
||||
ici (``AI_KEYS_FILE`` garde son chemin relatif ``data/api_keys.json``,
|
||||
résolu depuis le même CWD au runtime).
|
||||
"""
|
||||
|
||||
import json as _json
|
||||
import logging
|
||||
import os
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from backend.ai import PROVIDERS, _read_ai_keys, get_ai_key
|
||||
from backend.auth.middleware import check_vault_access, require_admin, require_auth
|
||||
from backend.indexer import index
|
||||
from backend.media_types import IMAGE_EXTENSIONS
|
||||
from backend.schemas import (
|
||||
AIKeyDeleteResponse,
|
||||
AIKeysResponse,
|
||||
AIModelsResponse,
|
||||
AITestResponse,
|
||||
AppConfigResponse,
|
||||
DashboardResponse,
|
||||
DiagnosticsResponse,
|
||||
StatusResponse,
|
||||
)
|
||||
from backend.search_executor import get_search_executor
|
||||
from backend.tools.secrets import (
|
||||
TOOL_KEY_NAMES as _TOOL_KEY_NAMES,
|
||||
)
|
||||
from backend.tools.secrets import (
|
||||
delete_tool_key as _delete_tool_key,
|
||||
)
|
||||
from backend.tools.secrets import (
|
||||
get_tool_key as _get_tool_key,
|
||||
)
|
||||
from backend.tools.secrets import (
|
||||
mask_value as _mask_tool_value,
|
||||
)
|
||||
from backend.tools.secrets import (
|
||||
set_tool_key as _set_tool_key,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["System"])
|
||||
|
||||
_BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
_CONFIG_PATH = _BASE_DIR / "data" / "config.json"
|
||||
|
||||
_DEFAULT_CONFIG = {
|
||||
"search_workers": 2,
|
||||
"debounce_ms": 300,
|
||||
"results_per_page": 50,
|
||||
"min_query_length": 2,
|
||||
"search_timeout_ms": 30000,
|
||||
"max_content_size": 100000,
|
||||
"snippet_context_chars": 120,
|
||||
"max_snippet_highlights": 5,
|
||||
"title_boost": 3.0,
|
||||
"path_boost": 1.5,
|
||||
"watcher_enabled": True,
|
||||
"watcher_use_polling": False,
|
||||
"watcher_polling_interval": 5.0,
|
||||
"watcher_debounce": 2.0,
|
||||
"tag_boost": 2.0,
|
||||
"prefix_max_expansions": 50,
|
||||
"recent_files_limit": 20,
|
||||
"max_backups_per_file": 10,
|
||||
"ai_default_provider": "deepseek",
|
||||
"ai_default_models": {},
|
||||
}
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
"""Load config from disk, merging with defaults."""
|
||||
config = dict(_DEFAULT_CONFIG)
|
||||
if _CONFIG_PATH.exists():
|
||||
try:
|
||||
stored = _json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
config.update(stored)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read config.json: {e}")
|
||||
return config
|
||||
|
||||
|
||||
def _save_config(config: dict) -> None:
|
||||
"""Persist config to disk."""
|
||||
try:
|
||||
_CONFIG_PATH.write_text(
|
||||
_json.dumps(config, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write config.json: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save config: {e}")
|
||||
|
||||
|
||||
AI_KEYS_FILE = Path("data/api_keys.json")
|
||||
|
||||
def _write_ai_keys(data: dict):
|
||||
AI_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = AI_KEYS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(_json.dumps(data, indent=2), encoding="utf-8")
|
||||
tmp.replace(AI_KEYS_FILE)
|
||||
|
||||
@router.get("/api/config", response_model=AppConfigResponse)
|
||||
async def api_get_config(current_user=Depends(require_auth)):
|
||||
"""Return current configuration with defaults for missing keys."""
|
||||
return _load_config()
|
||||
|
||||
|
||||
@router.post("/api/config", response_model=AppConfigResponse)
|
||||
async def api_set_config(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Update configuration. Only known keys are accepted.
|
||||
|
||||
Keys matching ``_DEFAULT_CONFIG`` are validated and persisted.
|
||||
Unknown keys are silently ignored.
|
||||
Returns the full merged config after update.
|
||||
"""
|
||||
current = _load_config()
|
||||
updated_keys = []
|
||||
for key, value in body.items():
|
||||
if key in _DEFAULT_CONFIG:
|
||||
expected_type = type(_DEFAULT_CONFIG[key])
|
||||
if isinstance(value, expected_type) or (expected_type is float and isinstance(value, (int, float))):
|
||||
current[key] = value
|
||||
updated_keys.append(key)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid type for '{key}': expected {expected_type.__name__}, got {type(value).__name__}",
|
||||
)
|
||||
_save_config(current)
|
||||
if any(k.startswith("ai_") for k in updated_keys):
|
||||
try:
|
||||
from backend.ai import reload_ai_config
|
||||
reload_ai_config()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to reload AI config: {e}")
|
||||
logger.info(f"Config updated: {updated_keys}")
|
||||
return current
|
||||
|
||||
|
||||
@router.get("/api/config/ai-keys", response_model=AIKeysResponse)
|
||||
async def api_get_ai_keys(current_user=Depends(require_admin)):
|
||||
"""Return stored AI keys (values masked)."""
|
||||
keys = _read_ai_keys()
|
||||
masked = {}
|
||||
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
|
||||
val = keys.get(k, "") or os.environ.get(k, "")
|
||||
if val:
|
||||
masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***"
|
||||
else:
|
||||
masked[k] = ""
|
||||
return masked
|
||||
|
||||
@router.post("/api/config/ai-keys", response_model=StatusResponse)
|
||||
async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}"""
|
||||
keys = _read_ai_keys()
|
||||
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
|
||||
if body.get(k):
|
||||
keys[k] = body[k]
|
||||
_write_ai_keys(keys)
|
||||
logger.info("AI keys updated")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.delete("/api/config/ai-keys/{provider_env}", response_model=AIKeyDeleteResponse)
|
||||
async def api_delete_ai_key(provider_env: str, current_user=Depends(require_admin)):
|
||||
"""Delete a specific AI provider key from storage."""
|
||||
allowed = {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY",
|
||||
"NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"}
|
||||
key_name = provider_env.upper()
|
||||
if key_name not in allowed:
|
||||
raise HTTPException(status_code=400, detail=f"Clé inconnue: {provider_env}")
|
||||
keys = _read_ai_keys()
|
||||
if key_name in keys:
|
||||
del keys[key_name]
|
||||
_write_ai_keys(keys)
|
||||
# Also clear from env at runtime so get_ai_key() no longer finds it
|
||||
os.environ.pop(key_name, None)
|
||||
logger.info(f"AI key deleted: {key_name}")
|
||||
return {"status": "deleted", "key": key_name}
|
||||
|
||||
|
||||
@router.get("/api/config/tool-keys", response_model=AIKeysResponse)
|
||||
async def api_get_tool_keys(current_user=Depends(require_admin)):
|
||||
"""Return tool/connected-source configuration (tokens masked, URLs clear)."""
|
||||
masked = {}
|
||||
for name in _TOOL_KEY_NAMES:
|
||||
masked[name] = _mask_tool_value(name, _get_tool_key(name))
|
||||
return masked
|
||||
|
||||
|
||||
@router.post("/api/config/tool-keys", response_model=StatusResponse)
|
||||
async def api_set_tool_keys(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Save tool/connected-source keys.
|
||||
|
||||
Only whitelisted names (``backend.tools.secrets.TOOL_KEY_NAMES``) are
|
||||
accepted: Tavily/Brave/SerpAPI/Exa API keys, Gitea URL + token, GitHub
|
||||
token. Empty values delete the stored entry.
|
||||
"""
|
||||
updated = []
|
||||
for name, value in body.items():
|
||||
if name not in _TOOL_KEY_NAMES:
|
||||
raise HTTPException(status_code=400, detail=f"Clé inconnue: {name}")
|
||||
if value is not None and not isinstance(value, str):
|
||||
raise HTTPException(status_code=400, detail=f"Type invalide pour {name}")
|
||||
_set_tool_key(name, value or "")
|
||||
updated.append(name)
|
||||
logger.info(f"Tool keys updated: {updated}")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.delete("/api/config/tool-keys/{name}", response_model=AIKeyDeleteResponse)
|
||||
async def api_delete_tool_key(name: str, current_user=Depends(require_admin)):
|
||||
"""Delete a stored tool key (the environment fallback still applies)."""
|
||||
key_name = name.upper()
|
||||
try:
|
||||
existed = _delete_tool_key(key_name)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
logger.info(f"Tool key deleted: {key_name} (existed={existed})")
|
||||
return {"status": "deleted", "key": key_name}
|
||||
|
||||
|
||||
@router.post("/api/config/ai-keys/test", response_model=AITestResponse)
|
||||
async def api_test_ai_keys(current_user=Depends(require_admin)):
|
||||
"""Test which AI providers are configured.
|
||||
|
||||
Each provider has a dedicated (URL, header-name) test pair.
|
||||
- Most OpenAI-compatible APIs use `Authorization: Bearer KEY`
|
||||
- Xiaomi MiMo uses `api-key: KEY`
|
||||
- Gemini uses a query-string key
|
||||
"""
|
||||
results = {}
|
||||
for key_name, label, test_url_tmpl, header_name in [
|
||||
# OpenAI-compatible — Authorization: Bearer
|
||||
("DEEPSEEK_API_KEY", "deepseek", "https://api.deepseek.com/v1/models", "Authorization"),
|
||||
("OPENROUTER_API_KEY","openrouter", "https://openrouter.ai/api/v1/models", "Authorization"),
|
||||
("NVIDIA_API_KEY", "nvidia", "https://integrate.api.nvidia.com/v1/models", "Authorization"),
|
||||
("QWENCLOUD_API_KEY", "qwencloud", "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"),
|
||||
("MISTRAL_API_KEY", "mistral", "https://api.mistral.ai/v1/models", "Authorization"),
|
||||
# Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer)
|
||||
("XIAOMI_API_KEY", "xiaomi", "https://api.xiaomimimo.com/v1/models", "api-key"),
|
||||
# Gemini — key in query string
|
||||
("GEMINI_API_KEY", "gemini", "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None),
|
||||
]:
|
||||
key = get_ai_key(key_name)
|
||||
if not key:
|
||||
results[label] = "non configuré"
|
||||
continue
|
||||
try:
|
||||
url = test_url_tmpl.replace("{key}", key) if "{key}" in test_url_tmpl else test_url_tmpl
|
||||
if header_name:
|
||||
req = urllib.request.Request(url, headers={header_name: key})
|
||||
else:
|
||||
req = urllib.request.Request(url)
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
results[label] = "ok"
|
||||
except Exception as e:
|
||||
# Truncate the error to keep the response small.
|
||||
results[label] = "erreur: " + str(e)[:80]
|
||||
return results
|
||||
|
||||
|
||||
@router.get("/api/config/ai-models", response_model=AIModelsResponse)
|
||||
async def api_list_ai_models(provider: str = Query(...), current_user=Depends(require_admin)):
|
||||
"""List available models for a given AI provider.
|
||||
|
||||
Strategy:
|
||||
1. Try the provider's public models endpoint (OpenAI-compatible /v1/models or Gemini).
|
||||
2. If the network call fails (timeout, 4xx, 5xx, DNS, etc.), fall back to a
|
||||
curated static list of known-good models for that provider.
|
||||
3. Always return a non-empty list when the provider is known, so the UI
|
||||
dropdown is never empty.
|
||||
"""
|
||||
provider = provider.lower()
|
||||
|
||||
from backend.model_capabilities import get_capabilities_for_models
|
||||
from backend.provider_capabilities import remember_declared_capabilities
|
||||
|
||||
all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral")
|
||||
if provider not in all_providers:
|
||||
return {"models": [], "error": f"Unknown provider: {provider}", "source": "validation"}
|
||||
|
||||
key_name = f"{provider.upper()}_API_KEY"
|
||||
key = get_ai_key(key_name)
|
||||
if not key:
|
||||
# No key configured — return curated fallback list so the UI can
|
||||
# still show what WOULD be available once a key is set.
|
||||
fallback = _FALLBACK_MODELS.get(provider, [])
|
||||
return {"models": fallback, "source": "fallback",
|
||||
"capabilities": get_capabilities_for_models(provider, fallback),
|
||||
"note": "API key not configured — showing default model list"}
|
||||
|
||||
# Build URL
|
||||
if provider == "gemini":
|
||||
url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}"
|
||||
elif provider == "deepseek":
|
||||
url = "https://api.deepseek.com/v1/models"
|
||||
elif provider == "openrouter":
|
||||
url = "https://openrouter.ai/api/v1/models"
|
||||
elif provider == "nvidia":
|
||||
url = "https://integrate.api.nvidia.com/v1/models"
|
||||
elif provider == "qwencloud":
|
||||
url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models"
|
||||
elif provider == "xiaomi":
|
||||
# Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer).
|
||||
# Endpoint: https://api.xiaomimimo.com/v1/models
|
||||
url = "https://api.xiaomimimo.com/v1/models"
|
||||
models = [] # parsed below with the custom header
|
||||
elif provider == "mistral":
|
||||
url = "https://api.mistral.ai/v1/models"
|
||||
|
||||
try:
|
||||
if provider == "gemini":
|
||||
req = urllib.request.Request(url)
|
||||
elif provider == "xiaomi":
|
||||
# Xiaomi MiMo uses a dedicated api-key header.
|
||||
req = urllib.request.Request(url, headers={"api-key": key})
|
||||
else:
|
||||
req = urllib.request.Request(url, headers={"Authorization": "Bearer " + key})
|
||||
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = _json.loads(resp.read().decode())
|
||||
|
||||
if provider == "gemini":
|
||||
models = [m.get("name", "") for m in data.get("models", []) if m.get("name")]
|
||||
# Gemini returns names like "models/gemini-1.5-flash" — strip prefix
|
||||
models = [m.replace("models/", "") for m in models]
|
||||
else:
|
||||
models = [m.get("id", "") for m in data.get("data", []) if m.get("id")]
|
||||
|
||||
# Cache the capabilities the provider declares for these models
|
||||
# (BUG-044) — get_capabilities_for_models() below then returns the
|
||||
# provider's own truth for the flags it declares, the curated table
|
||||
# for the rest. Providers that declare nothing are left untouched.
|
||||
remember_declared_capabilities(provider, data)
|
||||
|
||||
if models:
|
||||
# Prepend the configured default if not already present
|
||||
default = PROVIDERS.get(provider, {}).get("model")
|
||||
if default and default not in models:
|
||||
models = [default] + models
|
||||
return {"models": models, "source": "live", "count": len(models),
|
||||
"capabilities": get_capabilities_for_models(provider, models)}
|
||||
# Empty list from API — fall through to fallback
|
||||
raise ValueError("empty model list from provider API")
|
||||
except Exception as e:
|
||||
# Network error, auth error, parsing error — use curated fallback
|
||||
fallback = _FALLBACK_MODELS.get(provider, [])
|
||||
return {"models": fallback, "source": "fallback", "error": str(e)[:200],
|
||||
"capabilities": get_capabilities_for_models(provider, fallback),
|
||||
"note": "Could not reach provider API — showing default model list"}
|
||||
|
||||
|
||||
# ── Curated fallback model lists ──────────────────────────────────────────
|
||||
# Used when the provider API is unreachable or returns empty.
|
||||
# Keep these short and focused on models known to work with the
|
||||
# OpenAI-compatible chat completions interface (or Gemini's generateContent).
|
||||
_FALLBACK_MODELS: dict[str, list[str]] = {
|
||||
"deepseek": [
|
||||
"deepseek-chat",
|
||||
"deepseek-reasoner",
|
||||
],
|
||||
"openrouter": [
|
||||
"openai/gpt-4o-mini",
|
||||
"openai/gpt-4o",
|
||||
"anthropic/claude-3.5-sonnet",
|
||||
"anthropic/claude-3-haiku",
|
||||
"google/gemini-2.0-flash-exp:free",
|
||||
"meta-llama/llama-3.1-70b-instruct",
|
||||
"meta-llama/llama-3.1-8b-instruct:free",
|
||||
"mistralai/mistral-large-latest",
|
||||
],
|
||||
"gemini": [
|
||||
"gemini-2.0-flash",
|
||||
"gemini-2.0-flash-exp",
|
||||
"gemini-1.5-pro",
|
||||
"gemini-1.5-flash",
|
||||
"gemini-1.5-flash-8b",
|
||||
],
|
||||
"nvidia": [
|
||||
"meta/llama-3.1-405b-instruct",
|
||||
"meta/llama-3.1-70b-instruct",
|
||||
"meta/llama-3.1-8b-instruct",
|
||||
"mistralai/mistral-large",
|
||||
"google/gemma-2-27b-it",
|
||||
"nvidia/llama-3.1-nemotron-70b-instruct",
|
||||
],
|
||||
"qwencloud": [
|
||||
"qwen-max",
|
||||
"qwen-plus",
|
||||
"qwen-turbo",
|
||||
"qwen-long",
|
||||
"qwen-vl-max",
|
||||
"qwen-vl-plus",
|
||||
],
|
||||
"xiaomi": [
|
||||
# Xiaomi MiMo models — the public /v1/models endpoint requires the
|
||||
# `api-key` custom header (NOT Authorization: Bearer), so the live
|
||||
# call often fails with 401 even with the right key. We ship a
|
||||
# known-good list as fallback. See https://mimo.mi.com/docs/
|
||||
"mimo-v2.5-pro",
|
||||
"mimo-v2.5",
|
||||
"mimo-v2.5-asr",
|
||||
"mimo-v2.5-tts",
|
||||
"mimo-v2.5-tts-voiceclone",
|
||||
"mimo-v2.5-tts-voicedesign",
|
||||
],
|
||||
"mistral": [
|
||||
"mistral-large-latest",
|
||||
"mistral-medium-latest",
|
||||
"mistral-small-latest",
|
||||
"open-mistral-7b",
|
||||
"open-mixtral-8x7b",
|
||||
"codestral-latest",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/diagnostics", response_model=DiagnosticsResponse)
|
||||
async def api_diagnostics(current_user=Depends(require_admin)):
|
||||
"""Return index statistics and system diagnostics.
|
||||
|
||||
Includes document counts, token counts, memory estimates,
|
||||
and inverted index status.
|
||||
"""
|
||||
import sys
|
||||
|
||||
from backend.search import get_inverted_index
|
||||
|
||||
inv = get_inverted_index()
|
||||
|
||||
# Per-vault stats
|
||||
vault_stats = {}
|
||||
total_files = 0
|
||||
total_tags = 0
|
||||
# Snapshot both dicts first: the indexer mutates them from background
|
||||
# threads, and iterating a live dict raises "dictionary changed size".
|
||||
for vname, vdata in list(index.items()):
|
||||
file_count = len(vdata.get("files", []))
|
||||
tag_count = len(vdata.get("tags", {}))
|
||||
vault_stats[vname] = {"file_count": file_count, "tag_count": tag_count}
|
||||
total_files += file_count
|
||||
total_tags += tag_count
|
||||
|
||||
# Memory estimate for inverted index
|
||||
word_index = inv.word_index.copy()
|
||||
word_index_entries = sum(len(docs) for docs in word_index.values())
|
||||
mem_estimate_mb = round(
|
||||
(sys.getsizeof(inv.word_index) + word_index_entries * 80
|
||||
+ len(inv.doc_info) * 200
|
||||
+ len(inv._sorted_tokens) * 60) / (1024 * 1024), 2
|
||||
)
|
||||
|
||||
return {
|
||||
"index": {
|
||||
"total_files": total_files,
|
||||
"total_tags": total_tags,
|
||||
"vaults": vault_stats,
|
||||
},
|
||||
"inverted_index": {
|
||||
"unique_tokens": len(word_index),
|
||||
"total_postings": word_index_entries,
|
||||
"documents": inv.doc_count,
|
||||
"sorted_tokens": len(inv._sorted_tokens),
|
||||
"is_ready": inv.is_ready(),
|
||||
"memory_estimate_mb": mem_estimate_mb,
|
||||
},
|
||||
"config": _load_config(),
|
||||
"search_executor": {
|
||||
"active": get_search_executor() is not None,
|
||||
"max_workers": get_search_executor()._max_workers if get_search_executor() else 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/dashboard", response_model=DashboardResponse)
|
||||
async def api_dashboard(current_user=Depends(require_auth)):
|
||||
"""Aggregated dashboard statistics across all accessible vaults."""
|
||||
vault_stats = []
|
||||
total_files = 0
|
||||
total_tags = set()
|
||||
total_size = 0
|
||||
total_images = 0
|
||||
for vname, vdata in index.items():
|
||||
if not check_vault_access(vname, current_user):
|
||||
continue
|
||||
files = vdata.get("files", [])
|
||||
fc = len(files)
|
||||
total_files += fc
|
||||
vtags = set()
|
||||
vsize = 0
|
||||
vimages = 0
|
||||
for f in files:
|
||||
vtags.update(f.get("tags", []))
|
||||
vsize += f.get("size", 0)
|
||||
if (f.get("extension") or "").lower() in IMAGE_EXTENSIONS:
|
||||
vimages += 1
|
||||
total_tags.update(vtags)
|
||||
total_size += vsize
|
||||
total_images += vimages
|
||||
vault_stats.append({
|
||||
"name": vname, "file_count": fc, "tag_count": len(vtags),
|
||||
"total_size_bytes": vsize, "image_count": vimages,
|
||||
})
|
||||
return {
|
||||
"vaults": vault_stats,
|
||||
"total_files": total_files,
|
||||
"total_tags": len(total_tags),
|
||||
"total_size_bytes": total_size,
|
||||
"total_images": total_images,
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Syncthing conflict endpoints (ROADMAP #85, tranche 8).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/conflicts*``), mêmes modèles de
|
||||
réponse, mêmes dépendances d'authentification.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_resolve_safe_path`` / ``_backup_file`` → :mod:`backend.services.paths`
|
||||
et :mod:`backend.services.backups` (pass-through).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
|
||||
from backend.audit import log_file_delete
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.indexer import get_conflicts, get_vault_data, remove_single_file
|
||||
from backend.schemas import ConflictResolveResponse, ConflictsResponse
|
||||
from backend.services.backups import create_backup
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.sse import sse_manager
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["conflicts"])
|
||||
|
||||
|
||||
@router.get("/api/conflicts", response_model=ConflictsResponse)
|
||||
async def api_conflicts(current_user=Depends(require_auth)):
|
||||
"""List sync-conflict files across accessible vaults."""
|
||||
all_conflicts = get_conflicts()
|
||||
# #194 : filtrage via check_vault_access ("*" n'inclut pas les homes).
|
||||
all_conflicts = [c for c in all_conflicts if check_vault_access(c["vault"], current_user)]
|
||||
return {"conflicts": all_conflicts, "total": len(all_conflicts)}
|
||||
|
||||
|
||||
@router.post("/api/conflicts/resolve", response_model=ConflictResolveResponse)
|
||||
async def api_conflict_resolve(body: dict = Body(...), current_user=Depends(require_auth)):
|
||||
"""Resolve a conflict: keep_local (delete conflict file) or keep_conflict (replace original)."""
|
||||
vault_name = body.get("vault")
|
||||
conflict_path = body.get("conflict_path")
|
||||
original_path = body.get("original_path")
|
||||
action = body.get("action") # "keep_local" or "keep_conflict"
|
||||
# mypy: narrow down from dict values
|
||||
assert isinstance(vault_name, str), "'vault' is required and must be a string"
|
||||
assert isinstance(conflict_path, str), "'conflict_path' is required and must be a string"
|
||||
assert isinstance(original_path, str), "'original_path' is required and must be a string"
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(403, f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(404, "Vault not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
conf_file = resolve_safe_path(vault_root, conflict_path)
|
||||
orig_file = resolve_safe_path(vault_root, original_path)
|
||||
if not conf_file.exists():
|
||||
raise HTTPException(404, "Conflict file not found")
|
||||
try:
|
||||
if action == "keep_conflict":
|
||||
create_backup(orig_file, vault_name, original_path)
|
||||
shutil.copy2(conf_file, orig_file)
|
||||
logger.info(f"Conflict resolved (keep_conflict): {conflict_path} → {original_path}")
|
||||
conf_file.unlink()
|
||||
await remove_single_file(vault_name, conflict_path)
|
||||
log_file_delete(current_user["username"], vault_name, conflict_path)
|
||||
await sse_manager.broadcast("file_deleted", {"vault": vault_name, "path": conflict_path})
|
||||
return {"status": "resolved", "action": action}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Error resolving conflict: {e!s}")
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Duplicate detection & merge endpoints (#166).
|
||||
|
||||
Read endpoints require vault access; the merge endpoint is destructive
|
||||
(backup first in the service layer) and additionally requires the
|
||||
confirmation token pattern used by mutating routes — here enforced by an
|
||||
explicit ``confirm=true`` body flag, mirroring the agent two-step flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.services import duplicates as _duplicates
|
||||
from backend.services.errors import ServiceError
|
||||
|
||||
router = APIRouter(prefix="/api/duplicates", tags=["duplicates"])
|
||||
|
||||
|
||||
class DuplicatePair(BaseModel):
|
||||
"""One candidate duplicate pair."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
file_a: str = Field(description="First file (vault-relative)")
|
||||
file_b: str = Field(description="Second file (vault-relative)")
|
||||
score: float = Field(description="Blended similarity in [0, 1]")
|
||||
|
||||
|
||||
class DuplicatesResponse(BaseModel):
|
||||
"""Response for GET /api/duplicates."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
vault: str = Field(description="Vault name")
|
||||
threshold: float = Field(description="Applied threshold")
|
||||
files_scanned: int = Field(description="Markdown files compared")
|
||||
truncated: bool = Field(description="True when the scan hit the file cap")
|
||||
pairs: list[DuplicatePair] = Field(description="Candidate pairs, best score first")
|
||||
|
||||
|
||||
class MergeResponse(BaseModel):
|
||||
"""Response for POST /api/duplicates/merge."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
strategy: str = Field(description="Applied merge strategy")
|
||||
target: str = Field(description="Surviving note")
|
||||
deleted: str = Field(description="Absorbed note (deleted after merge)")
|
||||
|
||||
|
||||
@router.get("", response_model=DuplicatesResponse)
|
||||
async def api_duplicates_list(
|
||||
vault: str = Query(..., description="Vault name"),
|
||||
threshold: float = Query(0.75, ge=0.3, le=1.0, description="Minimum similarity"),
|
||||
limit: int = Query(20, ge=1, le=200, description="Max pairs"),
|
||||
subdir: str = Query("", description="Directory scope"),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""List candidate duplicate notes ordered by descending score."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
try:
|
||||
return _duplicates.find_duplicate_pairs(vault, threshold=threshold, limit=limit, subdir=subdir)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(e.status or 400, e.message) from e
|
||||
|
||||
|
||||
@router.post("/merge", response_model=MergeResponse)
|
||||
async def api_duplicates_merge(
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Merge *source_path* into *target_path* (``confirm: true`` required)."""
|
||||
vault = str(body.get("vault") or "")
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
if body.get("confirm") is not True:
|
||||
raise HTTPException(400, "Fusion destructive : confirmez avec {confirm: true}")
|
||||
try:
|
||||
return _duplicates.merge_duplicates(
|
||||
vault,
|
||||
str(body.get("source_path") or ""),
|
||||
str(body.get("target_path") or ""),
|
||||
strategy=str(body.get("strategy") or "append"),
|
||||
)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(e.status or 400, e.message) from e
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Endpoints des références croisées des notes — Fusion #215 (ACC-1.3).
|
||||
|
||||
Création par collage d'une URL profonde d'une autre application
|
||||
(Flowdeck, TikTak, Kanbante, ObsiGate) — zéro couplage serveur :
|
||||
l'URL est reconnue par les formes figées du contrat
|
||||
``dracodev-commons/contrats/liens-profonds.md`` (voir
|
||||
:mod:`backend.external_refs`), la référence est stockée chez la note
|
||||
propriétaire et affichée dans sa section « Liens ».
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.external_refs import add_ref, delete_ref, get_ref, list_refs, parse_deep_link
|
||||
from backend.schemas import ExternalRef, ExternalRefCreateRequest, ExternalRefsResponse
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["ExternalRefs"])
|
||||
|
||||
_FORMES_ACCEPTEES = (
|
||||
"URL non reconnue — formes acceptées : TikTak ?task= ou ?list=, "
|
||||
"ObsiGate #file=, Flowdeck /pages/ ou /collections/, "
|
||||
"issue Gitea (carte Kanbante)"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/refs", response_model=ExternalRefsResponse)
|
||||
async def api_list_refs(
|
||||
vault: str = Query(...),
|
||||
path: str = Query(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Références croisées posées sur une note."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(status_code=403, detail="Accès au vault refusé")
|
||||
username = current_user.get("username") or ""
|
||||
return {"refs": list_refs(username, vault, path)}
|
||||
|
||||
|
||||
@router.post("/api/refs", response_model=ExternalRef, status_code=201)
|
||||
async def api_add_ref(
|
||||
payload: ExternalRefCreateRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Crée une référence sur une note à partir d'une URL profonde collée."""
|
||||
if not check_vault_access(payload.vault, current_user):
|
||||
raise HTTPException(status_code=403, detail="Accès au vault refusé")
|
||||
parsed = parse_deep_link(payload.url)
|
||||
if parsed is None:
|
||||
raise HTTPException(status_code=422, detail=_FORMES_ACCEPTEES)
|
||||
target_app, target_type, target_id, default_label = parsed
|
||||
label = (payload.label or "").strip() or default_label
|
||||
username = current_user.get("username") or ""
|
||||
return add_ref(
|
||||
username,
|
||||
payload.vault,
|
||||
payload.path,
|
||||
target_app,
|
||||
target_type,
|
||||
target_id,
|
||||
payload.url.strip(),
|
||||
label,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/api/refs/{ref_id}", response_model=ExternalRef)
|
||||
async def api_delete_ref(
|
||||
ref_id: str,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Retire une référence d'une note."""
|
||||
username = current_user.get("username") or ""
|
||||
ref = get_ref(username, ref_id)
|
||||
if ref is None:
|
||||
raise HTTPException(status_code=404, detail="Lien introuvable")
|
||||
if not check_vault_access(ref["owner_vault"], current_user):
|
||||
raise HTTPException(status_code=403, detail="Accès au vault refusé")
|
||||
removed = delete_ref(username, ref_id)
|
||||
if removed is None: # course improbable : disparue entre get et delete
|
||||
raise HTTPException(status_code=404, detail="Lien introuvable")
|
||||
return removed
|
||||
@@ -0,0 +1,290 @@
|
||||
# backend/routers/file_chat.py — chat (#169, #190)
|
||||
"""Chat endpoints: history read + message post with SSE fan-out.
|
||||
|
||||
- ``GET/POST /api/file/{vault_name}/chat`` — per-file chat: auth + vault
|
||||
access + path traversal check (``resolve_safe_path`` raises
|
||||
``ServiceError`` mapped by the app-level handler).
|
||||
- ``GET/POST /api/chat`` — the **general chat** (#190), not bound to a file.
|
||||
- ``POST /api/chat/upload`` / ``GET /api/chat/attachment/{name}`` (#190):
|
||||
image/video attachments (extension allow-list, size cap, UUID name).
|
||||
|
||||
Every post is broadcast on the existing SSE channel (``chat_message``) so
|
||||
all connected clients update live without a second WebSocket.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from backend import file_chat as _store
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.auth.user_store import get_all_users, get_user
|
||||
from backend.indexer import get_vault_data
|
||||
from backend.schemas import ChatHistoryResponse, ChatMessageResponse, ChatReadResponse, StatusResponse
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.sse import sse_manager
|
||||
|
||||
router = APIRouter() # tags dérivés de `tag_for_path` → « Files »
|
||||
|
||||
|
||||
def _check(vault_name: str, path: str, current_user: dict[str, Any]) -> None:
|
||||
"""Authz + traversal guard shared by both verbs."""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(403, f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(404, "Vault not found")
|
||||
resolve_safe_path(Path(vault_data["path"]), path) # ServiceError → 403/500
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/chat", response_model=ChatHistoryResponse)
|
||||
async def api_file_chat_history(
|
||||
vault_name: str,
|
||||
path: str,
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Return the chat history for a file (chronological)."""
|
||||
_check(vault_name, path, current_user)
|
||||
return {"messages": _store.get_messages(vault_name, path), "read": _store.get_read(vault_name, path)}
|
||||
|
||||
|
||||
@router.post("/api/file/{vault_name}/chat", response_model=ChatMessageResponse)
|
||||
async def api_file_chat_post(
|
||||
vault_name: str,
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Post a chat message and broadcast it on SSE (``chat_message``)."""
|
||||
path = str(body.get("path") or "")
|
||||
text = str(body.get("text") or "")
|
||||
if not path:
|
||||
raise HTTPException(400, "path is required")
|
||||
if not text.strip():
|
||||
raise HTTPException(400, "text is required")
|
||||
_check(vault_name, path, current_user)
|
||||
msg = _store.add_message(vault_name, path, current_user.get("username", ""), text)
|
||||
await sse_manager.broadcast("chat_message", {"vault": vault_name, "path": path, "message": msg})
|
||||
return {"message": msg, "status": "ok"}
|
||||
|
||||
|
||||
# --- #190 : chat général ----------------------------------------------------
|
||||
|
||||
def _attachment(body: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Validate the optional ``attachment`` object sent by the client."""
|
||||
raw = body.get("attachment")
|
||||
if not raw or not isinstance(raw, dict):
|
||||
return None
|
||||
name = str(raw.get("name") or "")
|
||||
# Only an already-uploaded file (or an http(s) URL) may travel along.
|
||||
if not _store.attachment_path(name) and not str(raw.get("url", "")).startswith(("http://", "https://", "/api/")):
|
||||
raise HTTPException(400, "attachment inconnu")
|
||||
return {
|
||||
"name": name,
|
||||
"url": str(raw.get("url") or ""),
|
||||
"mime": str(raw.get("mime") or ""),
|
||||
"kind": str(raw.get("kind") or "file"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/chat", response_model=ChatHistoryResponse)
|
||||
async def api_chat_history(current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Return the general chat history (#190, chronological)."""
|
||||
return {
|
||||
"messages": _store.get_global_messages(),
|
||||
"read": _store.get_read(_store.GLOBAL_VAULT, _store.GLOBAL_PATH),
|
||||
}
|
||||
|
||||
|
||||
# --- #192 : accusé de réception ---------------------------------------------
|
||||
|
||||
def _check_read(vault: str, path: str, current_user: dict[str, Any]) -> None:
|
||||
"""Authorization for marking a conversation read (#192).
|
||||
|
||||
General chat is open to any member, a DM only to the two participants,
|
||||
a file chat follows the vault ACL.
|
||||
"""
|
||||
if vault == _store.GLOBAL_VAULT:
|
||||
return
|
||||
if vault == _store.DM_VAULT:
|
||||
if current_user.get("username") not in str(path).split("|"):
|
||||
raise HTTPException(403, "Accès refusé à cette conversation privée")
|
||||
return
|
||||
_check(vault, path, current_user)
|
||||
|
||||
|
||||
@router.post("/api/chat/read", response_model=ChatReadResponse)
|
||||
async def api_chat_read(
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Record that the caller has seen a conversation (#192).
|
||||
|
||||
Broadcast on SSE (``chat_read``) so the sender's own messages flip to
|
||||
✓✓ live on every connected client.
|
||||
"""
|
||||
vault = str(body.get("vault") or "")
|
||||
path = str(body.get("path") or "")
|
||||
if not vault or not path:
|
||||
raise HTTPException(400, "vault and path are required")
|
||||
_check_read(vault, path, current_user)
|
||||
username = current_user.get("username", "")
|
||||
read = _store.mark_read(vault, path, username)
|
||||
await sse_manager.broadcast(
|
||||
"chat_read",
|
||||
{"vault": vault, "path": path, "user": username, "ts": read.get(username, 0.0), "read": read},
|
||||
)
|
||||
return {"read": read, "status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/chat", response_model=ChatMessageResponse)
|
||||
async def api_chat_post(
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Post to the general chat and broadcast it on SSE (``chat_message``)."""
|
||||
text = str(body.get("text") or "")
|
||||
if not text.strip():
|
||||
raise HTTPException(400, "text is required")
|
||||
# #191 — best-effort link preview: a dead/slow URL never blocks the post.
|
||||
msg = _store.add_global_message(
|
||||
current_user.get("username", ""), text, _attachment(body),
|
||||
_store.build_preview(text),
|
||||
)
|
||||
await sse_manager.broadcast(
|
||||
"chat_message",
|
||||
{"vault": _store.GLOBAL_VAULT, "path": _store.GLOBAL_PATH, "message": msg},
|
||||
)
|
||||
return {"message": msg, "status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/chat/upload")
|
||||
async def api_chat_upload(
|
||||
file: UploadFile = File(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Store an image/video attachment (#190). Returns ``{attachment}``."""
|
||||
data = await file.read()
|
||||
try:
|
||||
info = _store.save_attachment(file.filename or "", data)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
return {"attachment": info}
|
||||
|
||||
|
||||
@router.get("/api/chat/attachment/{name}")
|
||||
async def api_chat_attachment(name: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Serve an uploaded attachment (name validated against the allow-list)."""
|
||||
path = _store.attachment_path(name)
|
||||
if not path:
|
||||
raise HTTPException(404, "Attachment not found")
|
||||
info = _store._MIME_BY_EXT.get(path.suffix.lower(), "application/octet-stream")
|
||||
return FileResponse(str(path), media_type=info)
|
||||
|
||||
|
||||
# --- #191 : suppression + messages privés -----------------------------------
|
||||
|
||||
def _owner_or_admin(msg_user: str, current_user: dict[str, Any]) -> None:
|
||||
"""A post may be deleted by its author or by an admin."""
|
||||
if current_user.get("role") != "admin" and current_user.get("username") != msg_user:
|
||||
raise HTTPException(403, "Seul l'auteur ou un administrateur peut supprimer ce message")
|
||||
|
||||
|
||||
def _find_and_authorize(vault: str, path: str, message_id: str, current_user: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Locate *message_id* in the conversation and check the delete right."""
|
||||
for m in _store.get_messages(vault, path):
|
||||
if m.get("id") == message_id:
|
||||
_owner_or_admin(m.get("user", ""), current_user)
|
||||
return m
|
||||
raise HTTPException(404, "Message not found")
|
||||
|
||||
|
||||
@router.delete("/api/chat/{message_id}", response_model=StatusResponse)
|
||||
async def api_chat_delete(message_id: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Delete a message from the general chat (author or admin, #191)."""
|
||||
_find_and_authorize(_store.GLOBAL_VAULT, _store.GLOBAL_PATH, message_id, current_user)
|
||||
if not _store.delete_message(_store.GLOBAL_VAULT, _store.GLOBAL_PATH, message_id):
|
||||
raise HTTPException(404, "Message not found")
|
||||
await sse_manager.broadcast(
|
||||
"chat_deleted",
|
||||
{"vault": _store.GLOBAL_VAULT, "path": _store.GLOBAL_PATH, "id": message_id},
|
||||
)
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.get("/api/chat/users", response_model=list[dict[str, Any]])
|
||||
async def api_chat_users(current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Usernames available for a private conversation (chat DM picker, #191).
|
||||
|
||||
Every authenticated member may see who else is around — this is a
|
||||
self-hosted portal, not a directory that needs hiding.
|
||||
"""
|
||||
me = current_user.get("username", "")
|
||||
return [
|
||||
{"username": u.get("username", ""), "display_name": u.get("display_name") or u.get("username", "")}
|
||||
for u in get_all_users()
|
||||
if u.get("username") and u.get("username") != me
|
||||
]
|
||||
|
||||
|
||||
def _dm_peer(username: str, current_user: dict[str, Any]) -> str:
|
||||
"""Validate the DM peer exists and is not ourselves."""
|
||||
if not username or username == current_user.get("username"):
|
||||
raise HTTPException(400, "Destinataire invalide")
|
||||
if not get_user(username):
|
||||
raise HTTPException(404, "Utilisateur inconnu")
|
||||
return username
|
||||
|
||||
|
||||
@router.get("/api/chat/dm/{username}", response_model=ChatHistoryResponse)
|
||||
async def api_chat_dm_history(username: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Private history with *username* (#191)."""
|
||||
peer = _dm_peer(username, current_user)
|
||||
return {
|
||||
"messages": _store.get_dm_messages(current_user["username"], peer),
|
||||
"read": _store.get_read(_store.DM_VAULT, _store.dm_path(current_user["username"], peer)),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/chat/dm/{username}", response_model=ChatMessageResponse)
|
||||
async def api_chat_dm_post(
|
||||
username: str,
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Post a private message and broadcast it to both participants (#191)."""
|
||||
peer = _dm_peer(username, current_user)
|
||||
text = str(body.get("text") or "")
|
||||
if not text.strip():
|
||||
raise HTTPException(400, "text is required")
|
||||
msg = _store.add_dm_message(
|
||||
current_user["username"], peer, current_user.get("username", ""), text,
|
||||
_attachment(body), _store.build_preview(text),
|
||||
)
|
||||
# Same shape as the general chat so the client routes on vault/path.
|
||||
await sse_manager.broadcast(
|
||||
"chat_message",
|
||||
{"vault": _store.DM_VAULT, "path": _store.dm_path(current_user["username"], peer), "message": msg},
|
||||
)
|
||||
return {"message": msg, "status": "ok"}
|
||||
|
||||
|
||||
@router.delete("/api/chat/dm/{username}/{message_id}", response_model=StatusResponse)
|
||||
async def api_chat_dm_delete(
|
||||
username: str,
|
||||
message_id: str,
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Delete one private message (author or admin, #191)."""
|
||||
peer = _dm_peer(username, current_user)
|
||||
vault, path = _store.DM_VAULT, _store.dm_path(current_user["username"], peer)
|
||||
_find_and_authorize(vault, path, message_id, current_user)
|
||||
if not _store.delete_message(vault, path, message_id):
|
||||
raise HTTPException(404, "Message not found")
|
||||
await sse_manager.broadcast(
|
||||
"chat_deleted", {"vault": vault, "path": path, "id": message_id}
|
||||
)
|
||||
return {"status": "deleted"}
|
||||
@@ -0,0 +1,569 @@
|
||||
"""Media, PDF, export & vault-settings endpoints (ROADMAP #85, tranche 6c).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/file/*/pdf*``, ``/api/export/*``,
|
||||
``/api/guide/download``, ``/api/image/*``, ``/api/media*``,
|
||||
``/api/attachments/*``, ``/api/vaults/*/settings``, ``/api/vault/*/files``,
|
||||
``/api/vaults/settings/all``), mêmes modèles de réponse, mêmes dépendances
|
||||
d'authentification.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_resolve_safe_path`` → :mod:`backend.services.paths` (pass-through).
|
||||
- ``_render_markdown`` vient de :mod:`backend.render` (#85 T9, sans cycle
|
||||
d'import).
|
||||
- ``_resolve_export_target`` / ``_safe_export_name`` (export uniquement)
|
||||
sont définis ici ; ``stream_file_with_range`` vit dans
|
||||
:mod:`backend.routers.helpers` (partagé).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
|
||||
from backend.attachment_indexer import get_attachment_stats, rescan_vault_attachments
|
||||
from backend.auth.middleware import check_vault_access, require_admin, require_auth
|
||||
from backend.export import ExportError, export_epub, export_html, export_md_bundle
|
||||
from backend.history import record_open
|
||||
from backend.indexer import get_vault_data, index, parse_markdown_file
|
||||
from backend.media_thumbs import generate_thumbnail, is_decodable
|
||||
from backend.media_types import is_audio, is_image, is_video, media_mime_type
|
||||
from backend.render import _render_markdown
|
||||
from backend.routers.helpers import media_max_inline_bytes, stream_file_with_range
|
||||
from backend.schemas import (
|
||||
AllVaultSettingsResponse,
|
||||
AttachmentRescanResponse,
|
||||
AttachmentStatsResponse,
|
||||
PdfInfoResponse,
|
||||
VaultFilesResponse,
|
||||
VaultSettingsResponse,
|
||||
)
|
||||
from backend.secret_redactor import redact_file_content
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import list_all_files
|
||||
from backend.vault_settings import get_vault_setting, update_vault_setting
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
# Lazy import: WeasyPrint PDF export (requires GTK, may not be available everywhere)
|
||||
try:
|
||||
from backend.pdf_export import build_pdf_html, generate_pdf
|
||||
except Exception: # pragma: no cover - WeasyPrint/GTK missing
|
||||
generate_pdf = None # type: ignore[assignment]
|
||||
build_pdf_html = None # type: ignore[assignment]
|
||||
|
||||
logging.getLogger("obsigate").warning("PDF export unavailable (WeasyPrint/GTK not found)")
|
||||
|
||||
router = APIRouter() # pas de tags : assignation par chemin via openapi_docs.tag_for_path (comme avant)
|
||||
|
||||
|
||||
def _resolve_export_target(vault_name: str, path: str, current_user: dict) -> tuple[Path, Path]:
|
||||
"""Resolve a vault + relative path into (vault_root, absolute file path).
|
||||
|
||||
Enforces auth (vault access) and path traversal protection.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
target = resolve_safe_path(vault_root, path)
|
||||
return vault_root, target
|
||||
|
||||
|
||||
def _safe_export_name(name: str) -> str:
|
||||
"""ASCII-safe, filename-safe download name (falls back to 'document')."""
|
||||
cleaned = "".join(c for c in name if c.isascii() and (c.isalnum() or c in " _-.")).strip()
|
||||
return cleaned or "document"
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/file/{vault_name}/pdf",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/pdf": {}}, "description": "PDF document"}},
|
||||
)
|
||||
async def api_file_pdf(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Download a markdown file as PDF."""
|
||||
if generate_pdf is None:
|
||||
raise HTTPException(501, "PDF export unavailable (WeasyPrint/GTK not available)")
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(403, f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(404, f"Vault '{vault_name}' not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists():
|
||||
raise HTTPException(404, f"File not found: {path}")
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
raise HTTPException(500, "Cannot read file")
|
||||
record_open(current_user.get("username"), vault_name, path)
|
||||
raw = redact_file_content(raw, str(file_path))
|
||||
post = parse_markdown_file(raw)
|
||||
html = _render_markdown(post.content, vault_name, file_path)
|
||||
title = post.metadata.get("title", file_path.stem)
|
||||
pdf_html = build_pdf_html(html, str(title))
|
||||
pdf_bytes = generate_pdf(pdf_html, str(title))
|
||||
safe_name = "".join(c for c in str(title) if c.isascii() and (c.isalnum() or c in " _-.")).strip() or "document"
|
||||
return Response(content=pdf_bytes, media_type="application/pdf", headers={"Content-Disposition": f'attachment; filename="{safe_name}.pdf"'})
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/export/html",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"text/html": {}}, "description": "Standalone HTML file"}},
|
||||
)
|
||||
async def api_export_html(
|
||||
vault: str = Query(..., description="Vault name"),
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Export a markdown note as a standalone HTML file."""
|
||||
try:
|
||||
vault_root, target = _resolve_export_target(vault, path, current_user)
|
||||
html_bytes = export_html(vault_root, target)
|
||||
except ExportError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
record_open(current_user.get("username"), vault, path)
|
||||
safe_name = _safe_export_name(target.stem)
|
||||
return Response(
|
||||
content=html_bytes,
|
||||
media_type="text/html; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{safe_name}.html"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/export/md-bundle",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/zip": {}}, "description": "Markdown ZIP bundle"}},
|
||||
)
|
||||
async def api_export_md_bundle(
|
||||
vault: str = Query(..., description="Vault name"),
|
||||
path: str = Query(..., description="Relative path to directory or file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Export a directory (or single file) of markdown as a ZIP bundle."""
|
||||
try:
|
||||
vault_root, target = _resolve_export_target(vault, path, current_user)
|
||||
zip_bytes = export_md_bundle(vault_root, target)
|
||||
except ExportError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
safe_name = _safe_export_name(target.name)
|
||||
return Response(
|
||||
content=zip_bytes,
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="{safe_name}.zip"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/export/epub",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/epub+zip": {}}, "description": "ePub document"}},
|
||||
)
|
||||
async def api_export_epub(
|
||||
vault: str = Query(..., description="Vault name"),
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Export a markdown note as an ePub document."""
|
||||
try:
|
||||
vault_root, target = _resolve_export_target(vault, path, current_user)
|
||||
epub_bytes = export_epub(vault_root, target)
|
||||
except ExportError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
record_open(current_user.get("username"), vault, path)
|
||||
safe_name = _safe_export_name(target.stem)
|
||||
return Response(
|
||||
content=epub_bytes,
|
||||
media_type="application/epub+zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="{safe_name}.epub"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/guide/download",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/pdf": {}, "text/markdown": {}}}},
|
||||
)
|
||||
async def api_guide_download(
|
||||
format: str = Query("md", description="Download format: 'md' or 'pdf'"),
|
||||
lang: str = Query("fr", description="Guide language: 'fr' or 'en'"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Download the in-app user guide as Markdown or PDF (#105).
|
||||
|
||||
The document is generated from the live help modal in index.html resolved
|
||||
through the locale files, so it always mirrors exactly what the user sees.
|
||||
"""
|
||||
from backend.guide_export import get_guide_document
|
||||
|
||||
if format not in ("md", "pdf"):
|
||||
raise HTTPException(status_code=400, detail="format doit être 'md' ou 'pdf'")
|
||||
try:
|
||||
payload, media, fname = get_guide_document(format, lang)
|
||||
except Exception as e: # weasyprint/reportlab unavailable
|
||||
logger.exception("guide export failed")
|
||||
raise HTTPException(status_code=500, detail=f"Export impossible: {e}") from e
|
||||
return Response(
|
||||
content=payload,
|
||||
media_type=media,
|
||||
headers={"Content-Disposition": f'attachment; filename="{fname}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/pdf/stream", response_class=FileResponse)
|
||||
async def api_pdf_stream(
|
||||
request: Request,
|
||||
vault_name: str,
|
||||
path: str = Query(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Stream a PDF file with Content-Type: application/pdf for inline browser viewing.
|
||||
|
||||
Supports HTTP Range requests (206 Partial Content) so browsers can
|
||||
progressively render large PDFs in the native viewer.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
if file_path.suffix.lower() != ".pdf":
|
||||
raise HTTPException(status_code=400, detail="Not a PDF file")
|
||||
|
||||
return stream_file_with_range(file_path, request, "application/pdf")
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/pdf/info", response_model=PdfInfoResponse)
|
||||
async def api_pdf_info(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to PDF file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return PDF metadata (pages, title, author, size) without the document content.
|
||||
|
||||
Lets the UI display file info before loading a heavy PDF into the viewer.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
if file_path.suffix.lower() != ".pdf":
|
||||
raise HTTPException(status_code=400, detail="Not a PDF file")
|
||||
|
||||
from backend.pdf_reader import extract_pdf_metadata
|
||||
meta = extract_pdf_metadata(file_path)
|
||||
stat = file_path.stat()
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"pages": meta.get("pages", 0),
|
||||
"title": meta.get("title") or file_path.name,
|
||||
"author": meta.get("author", ""),
|
||||
"size_bytes": stat.st_size,
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/image/{vault_name}",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/octet-stream": {}}, "description": "Image bytes"}},
|
||||
)
|
||||
async def api_image(vault_name: str, path: str = Query(..., description="Relative path to image"), current_user=Depends(require_auth)):
|
||||
"""Serve an image file with proper MIME type.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
|
||||
Returns:
|
||||
Image file with appropriate content-type header.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"Image not found: {path}")
|
||||
|
||||
mime_type = media_mime_type(str(file_path))
|
||||
|
||||
# #108-B3 — a standalone SVG opened in a tab executes its embedded JS
|
||||
# (same-origin XSS). ``sandbox`` forces a unique opaque origin with no
|
||||
# script execution; inside an <img> tag the header is irrelevant.
|
||||
headers = {"X-Content-Type-Options": "nosniff"}
|
||||
if file_path.suffix.lower() == ".svg":
|
||||
headers["Content-Security-Policy"] = "sandbox"
|
||||
|
||||
try:
|
||||
# Read and return the image file
|
||||
content = file_path.read_bytes()
|
||||
return Response(content=content, media_type=mime_type, headers=headers)
|
||||
except PermissionError:
|
||||
raise HTTPException(status_code=403, detail="Permission denied")
|
||||
except Exception as e:
|
||||
logger.error(f"Error serving image {vault_name}/{path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error serving image: {e!s}")
|
||||
|
||||
|
||||
@router.get("/api/media/{vault_name}", response_class=FileResponse)
|
||||
async def api_media_stream(
|
||||
request: Request,
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to audio/video file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Stream an audio/video file with HTTP Range support (roadmap #109-A2).
|
||||
|
||||
Serves the bytes with the correct MIME type and honours ``Range`` requests
|
||||
(``206 Partial Content`` + ``Content-Range``/``Accept-Ranges``), which is
|
||||
what enables scrubbing in ``<audio>``/``<video>`` and is required by Safari
|
||||
for MP4. Files above ``OBSIGATE_MEDIA_MAX_INLINE_MB`` (default 500 MB) are
|
||||
refused with ``413`` — the viewer falls back to the download button.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"Media not found: {path}")
|
||||
|
||||
ext = file_path.suffix.lower()
|
||||
if not (is_audio(ext) or is_video(ext)):
|
||||
raise HTTPException(status_code=400, detail="Not an audio/video file")
|
||||
|
||||
if file_path.stat().st_size > media_max_inline_bytes():
|
||||
raise HTTPException(status_code=413, detail="Media too large for inline streaming")
|
||||
|
||||
return stream_file_with_range(file_path, request, media_mime_type(str(file_path)))
|
||||
|
||||
|
||||
@router.get("/api/media/{vault_name}/thumb", response_class=FileResponse)
|
||||
async def api_media_thumb(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to image"),
|
||||
size: int = Query(256, ge=32, le=1024, description="Max thumbnail edge in pixels"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Serve a cached WebP thumbnail of an image (roadmap #108-C).
|
||||
|
||||
SVG (and any format Pillow cannot decode) falls back to the original
|
||||
bytes. Generation runs in a thread and is capped at 2 s; on timeout or
|
||||
failure the original is served so the UI never breaks.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"Image not found: {path}")
|
||||
if not is_image(file_path.suffix.lower()):
|
||||
raise HTTPException(status_code=400, detail="Not an image file")
|
||||
|
||||
mime_type = media_mime_type(str(file_path))
|
||||
if not is_decodable(file_path):
|
||||
# SVG: never let a standalone navigation execute embedded JS (#108-B3).
|
||||
svg_headers = {"X-Content-Type-Options": "nosniff"}
|
||||
if file_path.suffix.lower() == ".svg":
|
||||
svg_headers["Content-Security-Policy"] = "sandbox"
|
||||
return FileResponse(str(file_path), media_type=mime_type, headers=svg_headers)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
thumb: Path | None = None
|
||||
try:
|
||||
thumb = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, generate_thumbnail, file_path, size),
|
||||
timeout=2.0,
|
||||
)
|
||||
except Exception:
|
||||
thumb = None
|
||||
|
||||
if thumb is not None and thumb.exists():
|
||||
return FileResponse(str(thumb), media_type="image/webp")
|
||||
return FileResponse(str(file_path), media_type=mime_type)
|
||||
|
||||
|
||||
@router.post("/api/attachments/rescan/{vault_name}", response_model=AttachmentRescanResponse)
|
||||
async def api_rescan_attachments(vault_name: str, current_user=Depends(require_admin)):
|
||||
"""Rescan attachments for a specific vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault to rescan.
|
||||
|
||||
Returns:
|
||||
Dict with status and attachment count.
|
||||
"""
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
vault_path = vault_data["path"]
|
||||
count = await rescan_vault_attachments(vault_name, vault_path)
|
||||
|
||||
logger.info(f"Rescanned attachments for vault '{vault_name}': {count} attachments")
|
||||
return {"status": "ok", "vault": vault_name, "attachment_count": count}
|
||||
|
||||
|
||||
@router.get("/api/attachments/stats", response_model=AttachmentStatsResponse)
|
||||
async def api_attachment_stats(vault: str | None = Query(None, description="Vault filter"), current_user=Depends(require_auth)):
|
||||
"""Get attachment statistics for vaults.
|
||||
|
||||
Args:
|
||||
vault: Optional vault name to filter stats.
|
||||
|
||||
Returns:
|
||||
Dict with vault names as keys and attachment counts as values.
|
||||
"""
|
||||
stats = get_attachment_stats(vault)
|
||||
return {"vaults": stats}
|
||||
|
||||
|
||||
@router.get("/api/vaults/{vault_name}/settings", response_model=VaultSettingsResponse)
|
||||
async def api_get_vault_settings(vault_name: str, current_user=Depends(require_auth)):
|
||||
"""Get UI display settings for a specific vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
|
||||
Returns:
|
||||
Dict with vault settings including hideHiddenFiles.
|
||||
"""
|
||||
if vault_name not in index:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
# Get persisted settings
|
||||
persisted = get_vault_setting(vault_name) or {}
|
||||
|
||||
# Default settings
|
||||
settings = {
|
||||
"hideHiddenFiles": False,
|
||||
}
|
||||
settings.update(persisted)
|
||||
|
||||
return settings
|
||||
|
||||
|
||||
@router.post("/api/vaults/{vault_name}/settings", response_model=VaultSettingsResponse)
|
||||
async def api_update_vault_settings(vault_name: str, body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Update UI display settings for a specific vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Dict with settings to update (hideHiddenFiles).
|
||||
|
||||
Returns:
|
||||
Updated settings dict.
|
||||
"""
|
||||
if vault_name not in index:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
# Validate settings
|
||||
settings_to_update = {}
|
||||
|
||||
if "hideHiddenFiles" in body:
|
||||
if not isinstance(body["hideHiddenFiles"], bool):
|
||||
raise HTTPException(status_code=400, detail="hideHiddenFiles must be a boolean")
|
||||
settings_to_update["hideHiddenFiles"] = body["hideHiddenFiles"]
|
||||
|
||||
# Update persisted settings
|
||||
try:
|
||||
updated = update_vault_setting(vault_name, settings_to_update)
|
||||
except PermissionError as e:
|
||||
logger.error(f"Permission error saving settings for vault '{vault_name}': {e}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Permission denied: Cannot write to settings file. Check /app/data permissions."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving settings for vault '{vault_name}': {e}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to save settings: {e!s}"
|
||||
)
|
||||
|
||||
logger.info(f"Updated settings for vault '{vault_name}': {settings_to_update}")
|
||||
|
||||
return updated
|
||||
|
||||
|
||||
@router.get("/api/vault/{vault_name}/files", response_model=VaultFilesResponse)
|
||||
async def api_vault_recent_files(
|
||||
vault_name: str,
|
||||
dir: str = Query("", description="Directory path within the vault (empty = root)"),
|
||||
limit: int = Query(200, description="Maximum number of files to return"),
|
||||
recursive: bool = Query(True, description="If true, list files recursively from directory and all subdirectories"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""List files in a vault directory sorted by modification time (newest first).
|
||||
|
||||
Returns file metadata suitable for a vault home page display.
|
||||
Unlike /api/browse, this endpoint sorts by mtime and returns
|
||||
additional metadata (size, modified time, extension).
|
||||
|
||||
When recursive=True (default), lists files from the directory
|
||||
AND all its subdirectories, with a ``rel_dir`` field indicating
|
||||
the subdirectory path relative to the requested directory.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
dir: Relative directory path within the vault (empty for root).
|
||||
limit: Maximum files to return (default 200).
|
||||
recursive: If true, recursively list files in subdirectories (default true).
|
||||
|
||||
Returns:
|
||||
JSON with vault, directory, count, recursive flag, and list of file entries.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
return list_all_files(vault_name, dir=dir, limit=limit, recursive=recursive)
|
||||
|
||||
|
||||
@router.get("/api/vaults/settings/all", response_model=AllVaultSettingsResponse)
|
||||
async def api_get_all_vault_settings(current_user=Depends(require_auth)):
|
||||
"""Get UI display settings for all vaults.
|
||||
|
||||
Returns:
|
||||
Dict mapping vault names to their settings.
|
||||
"""
|
||||
all_settings = {}
|
||||
|
||||
for vault_name in index:
|
||||
persisted = get_vault_setting(vault_name) or {}
|
||||
|
||||
settings = {
|
||||
"hideHiddenFiles": False,
|
||||
}
|
||||
settings.update(persisted)
|
||||
all_settings[vault_name] = settings
|
||||
|
||||
return all_settings
|
||||
@@ -0,0 +1,755 @@
|
||||
"""File browsing & reading endpoints (ROADMAP #85, tranche 6a).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/browse/*``, ``/api/file/*`` en
|
||||
lecture), mêmes modèles de réponse (déménagés dans
|
||||
:mod:`backend.schemas`), mêmes dépendances d'authentification.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_resolve_safe_path`` → :mod:`backend.services.paths` (pass-through).
|
||||
- ``_render_markdown`` vient de :mod:`backend.render` (#85 T9, sans cycle
|
||||
d'import).
|
||||
- ``_content_disposition`` / ``_media_max_inline_bytes`` / ``EXT_TO_LANG``
|
||||
ont déménagé : helpers partagés dans :mod:`backend.routers.helpers`
|
||||
(``EXT_TO_LANG`` n'était utilisé que par la vue fichier).
|
||||
"""
|
||||
|
||||
import html as html_mod
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.history import record_open
|
||||
from backend.indexer import (
|
||||
_extract_tags,
|
||||
get_backlinks,
|
||||
get_vault_data,
|
||||
parse_markdown_file,
|
||||
)
|
||||
from backend.media_types import is_audio, is_image, is_video, media_mime_type
|
||||
from backend.render import _render_markdown
|
||||
from backend.routers.helpers import media_max_inline_bytes
|
||||
from backend.schemas import (
|
||||
BacklinksResponse,
|
||||
BrowseResponse,
|
||||
FileContentResponse,
|
||||
FileRawResponse,
|
||||
XlsxDashboardResponse,
|
||||
XlsxSheetWindowResponse,
|
||||
)
|
||||
from backend.services.files import read_raw_file
|
||||
from backend.services.mutations import file_revision
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import browse_directory, chemin_exclu, get_vault_root
|
||||
from backend.share import list_shares
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
|
||||
def _resolve_shared_file(vault_name: str, path: str, username: str | None):
|
||||
"""#196 — resolve ``home-<user>/Partage/<file>`` to the real source file.
|
||||
|
||||
Returns ``(real_vault, real_path)`` when *path* is a received share
|
||||
mounted in the user's personal folder, else ``None``. Read-only: only a
|
||||
recipient (or the share creator, whose own file is already accessible)
|
||||
gets a mapping — a share directed to someone else never resolves here.
|
||||
|
||||
Chemin canonique : ``Partage/<token>/<nom>`` (token = lève l'ambiguïté
|
||||
de deux partages au même nom). Fallback par nom seul pour les liens
|
||||
ne transportant pas le token.
|
||||
"""
|
||||
if not username or not vault_name.startswith("home-") or not path.startswith("Partage/"):
|
||||
return None
|
||||
owner = vault_name[len("home-"):]
|
||||
if owner != username:
|
||||
return None
|
||||
# Chemin virtuel canonique : Partage/<token>/<nom> — le token lève
|
||||
# l'ambiguïté (deux partages, même nom de base). Fallback : match par
|
||||
# nom pour les liens ne portant pas le token.
|
||||
rest = path.split("/", 1)[1]
|
||||
parts = rest.split("/", 1)
|
||||
if len(parts) == 2 and len(parts[0]) >= 20: # token (64 hex) vs nom de fichier
|
||||
token, _name = parts
|
||||
for s in list_shares(user=username):
|
||||
if s.get("token") == token and s.get("created_by") != username:
|
||||
return s["vault"], s["path"]
|
||||
return None
|
||||
for s in list_shares(user=username):
|
||||
if s.get("created_by") == username:
|
||||
continue
|
||||
if (s.get("path") or "").split("/")[-1] == rest:
|
||||
return s["vault"], s["path"]
|
||||
return None
|
||||
|
||||
# Map file extensions to highlight.js language hints
|
||||
EXT_TO_LANG = {
|
||||
".py": "python", ".js": "javascript", ".ts": "typescript",
|
||||
".jsx": "jsx", ".tsx": "tsx", ".sh": "bash", ".bash": "bash",
|
||||
".zsh": "bash", ".fish": "fish", ".bat": "batch", ".cmd": "batch",
|
||||
".ps1": "powershell", ".json": "json", ".yaml": "yaml", ".yml": "yaml",
|
||||
".toml": "toml", ".xml": "xml", ".csv": "plaintext",
|
||||
".cfg": "ini", ".ini": "ini", ".conf": "ini", ".env": "bash",
|
||||
".html": "html", ".css": "css", ".scss": "scss", ".less": "less",
|
||||
".java": "java", ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
|
||||
".cs": "csharp", ".go": "go", ".rs": "rust", ".rb": "ruby",
|
||||
".php": "php", ".sql": "sql", ".r": "r", ".swift": "swift",
|
||||
".kt": "kotlin", ".txt": "plaintext", ".log": "plaintext",
|
||||
".lua": "lua", ".pl": "perl", ".pm": "perl", ".ex": "elixir", ".exs": "elixir",
|
||||
".dart": "dart", ".tf": "haskell", ".gradle": "groovy", ".groovy": "groovy",
|
||||
".graphql": "graphql", ".gql": "graphql", ".prisma": "sql", ".proto": "c",
|
||||
".vb": "basic", ".asm": "x86asm", ".s": "armasm",
|
||||
".vue": "xml", ".svelte": "xml", ".astro": "xml",
|
||||
".properties": "ini", ".service": "ini", ".hosts": "ini",
|
||||
".ksh": "bash", ".dockerfile": "dockerfile",
|
||||
".makefile": "makefile", ".cmake": "cmake",
|
||||
}
|
||||
|
||||
router = APIRouter(tags=["files"])
|
||||
|
||||
|
||||
@router.get("/api/browse/{vault_name}", response_model=BrowseResponse)
|
||||
async def api_browse(vault_name: str, path: str = "", current_user=Depends(require_auth)):
|
||||
"""Browse directories and files in a vault at a given path level.
|
||||
|
||||
Returns sorted entries (directories first, then files) with metadata.
|
||||
Hidden files/directories (starting with ``"."`` ) are excluded.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault to browse.
|
||||
path: Relative directory path within the vault (empty = root).
|
||||
|
||||
Returns:
|
||||
``BrowseResponse`` with vault name, path, and item list.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
return browse_directory(vault_name, path)
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/raw", response_model=FileRawResponse)
|
||||
async def api_file_raw(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Return raw file content as plain text.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
|
||||
Returns:
|
||||
``FileRawResponse`` with vault, path, and raw text content.
|
||||
"""
|
||||
shared = _resolve_shared_file(vault_name, path, current_user.get("username"))
|
||||
if shared:
|
||||
# Partage dirigé = autorisation (cf. api_file).
|
||||
vault_name, path = shared
|
||||
elif not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
return read_raw_file(vault_name, path)
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/download", response_class=FileResponse)
|
||||
async def api_file_download(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Download a file as an attachment.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
|
||||
Returns:
|
||||
``FileResponse`` with ``application/octet-stream`` content-type.
|
||||
"""
|
||||
shared = _resolve_shared_file(vault_name, path, current_user.get("username"))
|
||||
if shared:
|
||||
# Partage dirigé = autorisation (cf. api_file).
|
||||
vault_name, path = shared
|
||||
elif not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
if chemin_exclu(vault_name, path):
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
# Record history
|
||||
record_open(current_user.get("username"), vault_name, path)
|
||||
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=file_path.name,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}/backlinks", response_model=BacklinksResponse)
|
||||
async def api_file_backlinks(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Get backlinks (files linking to this file via wikilinks).
|
||||
|
||||
Returns a list of files that contain `[[wikilinks]]` pointing
|
||||
to the requested file, across all accessible vaults.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault containing the target file.
|
||||
path: Relative path of the target file within the vault.
|
||||
|
||||
Returns:
|
||||
``{"vault": str, "path": str, "backlinks": [...]}``
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
backlinks = get_backlinks(vault_name, path)
|
||||
|
||||
# Filter by user-accessible vaults (#194 : check_vault_access, "*" sans
|
||||
# les dossiers persos).
|
||||
backlinks = [b for b in backlinks if check_vault_access(b["vault"], current_user)]
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"backlinks": backlinks,
|
||||
"total": len(backlinks),
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/file/{vault_name}/xlsx/dashboard", response_model=XlsxDashboardResponse
|
||||
)
|
||||
def api_file_xlsx_dashboard(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .xlsx workbook"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return the dashboard metadata of an .xlsx workbook (#153 A17).
|
||||
|
||||
Named ranges (workbook- or sheet-scoped), chart/pivot object counts and
|
||||
per-sheet KPI stats (non-empty cells, rows/cols coverage, formulas,
|
||||
numeric cells, first numeric values as KPI cards). Read-only, bounded by
|
||||
the 500x40 render caps; never raises for an unreadable workbook — an
|
||||
empty payload comes back and the viewer hides the panel.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
if chemin_exclu(vault_name, path):
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
_vault_root = get_vault_root(vault_name)
|
||||
file_path = resolve_safe_path(_vault_root, path)
|
||||
if not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
if file_path.suffix.lower() not in (".xlsx", ".xlsm"):
|
||||
raise HTTPException(
|
||||
status_code=415, detail="Le fichier n'est pas un classeur .xlsx/.xlsm"
|
||||
)
|
||||
|
||||
from backend.xlsx_reader import read_workbook_dashboard
|
||||
|
||||
try:
|
||||
dashboard = read_workbook_dashboard(file_path)
|
||||
except Exception as e:
|
||||
logger.error(f"XLSX dashboard read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading XLSX: {e!s}")
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
**dashboard,
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/file/{vault_name}/xlsx/sheet", response_model=XlsxSheetWindowResponse
|
||||
)
|
||||
def api_file_xlsx_sheet(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .xlsx/.xlsm file"),
|
||||
sheet: str = Query(..., description="Sheet name (as shown in the viewer tab)"),
|
||||
offset: int = Query(0, ge=0, description="0-based index of the first row to return"),
|
||||
limit: int = Query(
|
||||
200, ge=1, le=1000, description="Rows to return (server-capped)"
|
||||
),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return a window of rows of one sheet of an .xlsx/.xlsm workbook (#153 A9).
|
||||
|
||||
Backs the viewer's lazy loading: instead of every sheet in a single JSON
|
||||
payload, the client asks for the block it is about to display. The row
|
||||
numbers and the ``data-cell`` references are the real A1 coordinates of the
|
||||
sheet, so a window behaves like the full render (editing a cell in it
|
||||
targets the right cell).
|
||||
|
||||
The response also carries ``total_rows``/``total_cols`` and the ``truncated``
|
||||
flag, so the client can say what is hidden behind the 500x40 render caps
|
||||
instead of silently hiding it.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path of the .xlsx file within the vault.
|
||||
sheet: Sheet name; **404** if the workbook has no such sheet.
|
||||
offset: 0-based index of the first row to return.
|
||||
limit: Rows to return, capped server-side at 1000.
|
||||
|
||||
Returns:
|
||||
``XlsxSheetWindowResponse`` with the rendered ``html`` of the window.
|
||||
|
||||
Raises:
|
||||
HTTPException: 403 (vault access), 404 (vault, file or sheet unknown),
|
||||
415 (not an .xlsx/.xlsm file), 500 (unreadable workbook).
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
if chemin_exclu(vault_name, path):
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
file_path = resolve_safe_path(Path(vault_data["path"]), path)
|
||||
if not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
# BUG-097 — a .xlsm rides the same editable viewer (and its lazy loading),
|
||||
# so its row windows must be servable too; other formats stay refused.
|
||||
if file_path.suffix.lower() not in (".xlsx", ".xlsm"):
|
||||
raise HTTPException(
|
||||
status_code=415, detail="Le fichier n'est pas un classeur .xlsx/.xlsm"
|
||||
)
|
||||
|
||||
# Import tardif : openpyxl n'est chargé que si un .xlsx est réellement demandé.
|
||||
from backend.xlsx_reader import read_sheet_window
|
||||
|
||||
try:
|
||||
window = read_sheet_window(file_path, sheet, offset=offset, limit=limit)
|
||||
except Exception as e:
|
||||
logger.error(f"XLSX sheet read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading XLSX: {e!s}")
|
||||
if window is None:
|
||||
raise HTTPException(status_code=404, detail=f"Feuille introuvable: {sheet}")
|
||||
|
||||
return {"vault": vault_name, "path": path, **window}
|
||||
|
||||
|
||||
@router.get("/api/file/{vault_name}", response_model=FileContentResponse)
|
||||
async def api_file(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Return rendered HTML and metadata for a file.
|
||||
|
||||
Markdown files are parsed for frontmatter, rendered with wikilink
|
||||
support, and returned with extracted tags. Other supported file
|
||||
types are syntax-highlighted as code blocks.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
|
||||
Returns:
|
||||
``FileContentResponse`` with HTML, metadata, and tags.
|
||||
"""
|
||||
# #196 — fichier reçu par partage dirigé : home-<user>/Partage/<fichier>
|
||||
# est résolu vers le fichier source (lecture seule, viewer standard).
|
||||
# Résolu AVANT l'ACL vault : le dossier est virtuel, l'autorisation réelle
|
||||
# est l'appartenance au partage (vérifiée dans le resolver).
|
||||
shared = _resolve_shared_file(vault_name, path, current_user.get("username"))
|
||||
if shared:
|
||||
# Le partage dirigé EST l'autorisation : le destinataire n'a par
|
||||
# définition pas accès au vault source — on ne passe PAS par l'ACL.
|
||||
vault_name, path = shared
|
||||
elif not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if not vault_data:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
if chemin_exclu(vault_name, path):
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
|
||||
# Record history
|
||||
record_open(current_user.get("username"), vault_name, path, title=file_path.name)
|
||||
|
||||
ext = file_path.suffix.lower()
|
||||
|
||||
# === PDF: special handling before read_text (binary file) ===
|
||||
if ext == ".pdf":
|
||||
try:
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text, extract_pdf_toc
|
||||
pdf_text = extract_pdf_text(file_path, max_chars=100000)
|
||||
pdf_meta = extract_pdf_metadata(file_path)
|
||||
pdf_toc = extract_pdf_toc(file_path)
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": pdf_meta.get("title") or file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": f"<div class='pdf-viewer'><p>PDF — {pdf_meta.get('pages', '?')} pages</p><pre>{pdf_text[:5000]}</pre></div>",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_pdf": True,
|
||||
"unsupported": False,
|
||||
"pdf_metadata": pdf_meta,
|
||||
"pdf_toc": pdf_toc,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"PDF read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading PDF: {e!s}")
|
||||
|
||||
# === Excel .xlsx: render sheets as HTML tables (binary, before read_text) ===
|
||||
if ext == ".xlsx":
|
||||
try:
|
||||
from backend.xlsx_reader import inspect_workbook, render_sheets
|
||||
|
||||
# #153 A15 — every sheet dict already carries its styles, aligns,
|
||||
# merges and freeze anchor (read_workbook_meta, one normal-mode
|
||||
# load inside render_sheets).
|
||||
sheets = render_sheets(file_path)
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": sheets[0]["html"] if sheets else "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_xlsx": True,
|
||||
"xlsx_sheets": sheets,
|
||||
# #156-A12 — optimistic-concurrency token: the viewer sends it
|
||||
# back as `if_match` so another writer cannot be overwritten in
|
||||
# silence (409 `conflict` instead).
|
||||
"xlsx_revision": file_revision(file_path),
|
||||
# #153 A1 — parts a save would drop; the viewer warns and asks
|
||||
# for an explicit confirmation before forcing the write.
|
||||
"xlsx_lossy_features": inspect_workbook(file_path),
|
||||
"unsupported": False,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"XLSX read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading XLSX: {e!s}")
|
||||
|
||||
# === Images: return as viewable image ===
|
||||
if is_image(ext):
|
||||
size = file_path.stat().st_size
|
||||
mime = media_mime_type(str(file_path))
|
||||
# #108-B1 — the raw endpoint returns JSON (FileRawResponse), so the
|
||||
# standalone <img> must point to /api/image, which serves the bytes
|
||||
# with the right MIME type. Paths are URL-encoded (accents, spaces).
|
||||
img_url = f"/api/image/{quote(vault_name, safe='')}?path={quote(path, safe='')}"
|
||||
html = (
|
||||
f'<div class="image-viewer">'
|
||||
f'<img src="{img_url}" '
|
||||
f'alt="{html_mod.escape(file_path.name, quote=True)}" '
|
||||
f'style="max-width:100%;max-height:80vh;object-fit:contain" />'
|
||||
f'</div>'
|
||||
)
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": html,
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_image": True,
|
||||
"image_mime": mime,
|
||||
"size_bytes": size,
|
||||
}
|
||||
|
||||
# === Audio / Video: HTML5 players streamed from /api/media (roadmap #109) ===
|
||||
if is_audio(ext) or is_video(ext):
|
||||
size = file_path.stat().st_size
|
||||
mime = media_mime_type(str(file_path))
|
||||
media_kind = "audio" if is_audio(ext) else "video"
|
||||
|
||||
# #109-A3 — beyond the inline limit the viewer falls back to download
|
||||
# (a single uvicorn worker must not be pinned by multi-GB media).
|
||||
if size > media_max_inline_bytes():
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"unsupported": True,
|
||||
"media_too_large": True,
|
||||
"size_bytes": size,
|
||||
}
|
||||
|
||||
# #109-A2 — byte-range endpoint: enables scrub and is required by Safari.
|
||||
stream_url = f"/api/media/{quote(vault_name, safe='')}?path={quote(path, safe='')}"
|
||||
if media_kind == "audio":
|
||||
html = (
|
||||
f'<div class="audio-viewer">'
|
||||
f'<audio controls preload="metadata" src="{stream_url}"></audio>'
|
||||
f'</div>'
|
||||
)
|
||||
else:
|
||||
html = (
|
||||
f'<div class="video-viewer">'
|
||||
f'<video controls playsinline preload="metadata" src="{stream_url}"></video>'
|
||||
f'</div>'
|
||||
)
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": html,
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_audio": media_kind == "audio",
|
||||
"is_video": media_kind == "video",
|
||||
"media_mime": mime,
|
||||
"stream_url": stream_url,
|
||||
"size_bytes": size,
|
||||
}
|
||||
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except PermissionError as e:
|
||||
logger.error(f"Permission denied reading file {path}: {e}")
|
||||
raise HTTPException(status_code=403, detail=f"Permission denied: cannot read file {path}")
|
||||
except UnicodeDecodeError:
|
||||
# Binary / unsupported file — return structured info with download option
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"unsupported": True,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error reading file {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading file: {e!s}")
|
||||
|
||||
# === Excel .xlsm: same editable viewer as .xlsx, macros preserved on save ===
|
||||
if ext == ".xlsm":
|
||||
try:
|
||||
from backend.xlsx_reader import inspect_workbook, render_sheets
|
||||
|
||||
sheets = render_sheets(file_path)
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": sheets[0]["html"] if sheets else "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_xlsx": True,
|
||||
"xlsx_sheets": sheets,
|
||||
"xlsx_revision": file_revision(file_path),
|
||||
# Macros are NOT lossy for .xlsm: keep_vba re-serializes them
|
||||
# (an empty LOSSY probe is what makes the save gate pass).
|
||||
"xlsx_lossy_features": [],
|
||||
"unsupported": False,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"XLSX read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading XLSX: {e!s}")
|
||||
|
||||
# === Legacy/ODF spreadsheets (.xls, .ods): read-only table view ===
|
||||
if ext in (".xls", ".ods"):
|
||||
try:
|
||||
from backend.xlsx_reader import render_legacy_workbook
|
||||
|
||||
sheets = render_legacy_workbook(file_path, ext)
|
||||
size = file_path.stat().st_size
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": sheets[0]["html"] if sheets else "",
|
||||
"raw_length": size,
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_xlsx": True,
|
||||
"xlsx_readonly": True,
|
||||
"xlsx_sheets": sheets,
|
||||
"unsupported": False,
|
||||
"size_bytes": size,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Spreadsheet read error for {path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reading spreadsheet: {e!s}")
|
||||
|
||||
# === CSV: spreadsheet-style table (same shape as the xlsx viewer) ===
|
||||
if ext == ".csv":
|
||||
from backend.xlsx_reader import render_csv_table
|
||||
|
||||
html = render_csv_table(raw)
|
||||
return {
|
||||
"vault": vault_name, "path": path,
|
||||
"title": file_path.name, "tags": [], "frontmatter": {},
|
||||
"html": html, "raw_length": len(raw), "extension": ext,
|
||||
"is_markdown": False, "is_csv": True,
|
||||
# #156-A12 — same stale-write guard as the workbooks.
|
||||
"xlsx_revision": file_revision(file_path),
|
||||
}
|
||||
|
||||
# === JSON: syntax-highlighted display ===
|
||||
if ext == ".json":
|
||||
import json as json_mod
|
||||
try:
|
||||
parsed = json_mod.loads(raw)
|
||||
formatted = json_mod.dumps(parsed, indent=2, ensure_ascii=False)
|
||||
except json_mod.JSONDecodeError:
|
||||
formatted = raw
|
||||
html = f"<pre class='json-viewer'><code>{html_mod.escape(formatted)}</code></pre>"
|
||||
return {
|
||||
"vault": vault_name, "path": path,
|
||||
"title": file_path.name, "tags": [], "frontmatter": {},
|
||||
"html": html, "raw_length": len(raw), "extension": ext,
|
||||
"is_markdown": False, "is_json": True,
|
||||
}
|
||||
|
||||
# === Excalidraw .excalidraw.md (Obsidian plugin format) ===
|
||||
if path.lower().endswith(".excalidraw.md"):
|
||||
import re as re_mod
|
||||
raw_lower = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
# Check for excalidraw-plugin in frontmatter or body
|
||||
if "excalidraw-plugin:" in raw_lower:
|
||||
# Extract compressed JSON block
|
||||
match = re_mod.search(r'```compressed-json\n(.*?)\n```', raw_lower, re_mod.DOTALL)
|
||||
if match:
|
||||
compressed = match.group(1).strip()
|
||||
return {
|
||||
"vault": vault_name, "path": path,
|
||||
"title": file_path.name.replace(".excalidraw.md", ""),
|
||||
"tags": [], "frontmatter": {},
|
||||
"html": "", "raw_length": len(raw_lower),
|
||||
"extension": ".excalidraw.md",
|
||||
"is_markdown": False,
|
||||
"is_excalidraw": True,
|
||||
"excalidraw_data_compressed": compressed,
|
||||
}
|
||||
# Fallback: treat as regular markdown
|
||||
raw = raw_lower
|
||||
if ext == ".excalidraw":
|
||||
import json as json_mod
|
||||
try:
|
||||
parsed = json_mod.loads(raw)
|
||||
except json_mod.JSONDecodeError:
|
||||
parsed = None
|
||||
if parsed and parsed.get("type") == "excalidraw":
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": parsed.get("appState", {}).get("name") or file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": "",
|
||||
"raw_length": len(raw),
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
"is_excalidraw": True,
|
||||
"excalidraw_data": {
|
||||
"elements": parsed.get("elements", []),
|
||||
"appState": parsed.get("appState", {}),
|
||||
"files": parsed.get("files", {}),
|
||||
},
|
||||
}
|
||||
else:
|
||||
# Not a valid Excalidraw file — fall through to text viewer
|
||||
pass
|
||||
|
||||
# === Plain text / other readable files ===
|
||||
TEXT_EXTENSIONS = {".txt", ".log", ".yml", ".yaml", ".toml", ".ini", ".cfg",
|
||||
".sh", ".bash", ".py", ".js", ".ts", ".html", ".css",
|
||||
".xml", ".rst", ".tex", ".sql", ".conf", ".env"}
|
||||
if ext in TEXT_EXTENSIONS or ext == ".md":
|
||||
pass # handled below or by markdown section
|
||||
|
||||
if ext == ".md":
|
||||
post = parse_markdown_file(raw)
|
||||
|
||||
# Extract metadata using shared indexer logic
|
||||
tags = _extract_tags(post)
|
||||
|
||||
title = post.metadata.get("title", file_path.stem.replace("-", " ").replace("_", " "))
|
||||
html_content = _render_markdown(post.content, vault_name, file_path, click_to_copy=True)
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": str(title),
|
||||
"tags": tags,
|
||||
"frontmatter": dict(post.metadata) if post.metadata else {},
|
||||
"html": html_content,
|
||||
"raw_length": len(raw),
|
||||
"extension": ext,
|
||||
"is_markdown": True,
|
||||
}
|
||||
else:
|
||||
# Non-markdown: wrap in syntax-highlighted code block
|
||||
lang = EXT_TO_LANG.get(ext, "")
|
||||
if not lang:
|
||||
# Fichiers sans extension usuels (Dockerfile, Makefile, etc.)
|
||||
NAME_TO_LANG = {
|
||||
"dockerfile": "dockerfile", "makefile": "makefile",
|
||||
"cmakelists.txt": "cmake", "jenkinsfile": "groovy",
|
||||
"vagrantfile": "ruby", "rakefile": "ruby", "gemfile": "ruby",
|
||||
"procfile": "plaintext", "bashrc": "bash", "bash_profile": "bash",
|
||||
"zshrc": "bash", "profile": "bash", "gitignore": "plaintext",
|
||||
}
|
||||
lang = NAME_TO_LANG.get(file_path.name.lower(), "plaintext")
|
||||
escaped = html_mod.escape(raw)
|
||||
html_content = f'<pre><code class="language-{lang}">{escaped}</code></pre>'
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"title": file_path.name,
|
||||
"tags": [],
|
||||
"frontmatter": {},
|
||||
"html": html_content,
|
||||
"raw_length": len(raw),
|
||||
"extension": ext,
|
||||
"is_markdown": False,
|
||||
}
|
||||
@@ -0,0 +1,810 @@
|
||||
"""File & directory mutation endpoints (ROADMAP #85, tranche 6b).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``PUT/DELETE/PATCH/POST /api/file/*``,
|
||||
``/api/directory/*``, ``/api/move/*``, ``/api/vault/*/batch-upload``),
|
||||
mêmes modèles de requête/réponse (déménagés dans :mod:`backend.schemas`),
|
||||
mêmes dépendances d'authentification et mêmes effets de bord (audit, index
|
||||
incrémental, SSE, webhooks, plugins, historique).
|
||||
|
||||
La logique métier vit déjà dans :mod:`backend.services.mutations`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from backend import agent_journal
|
||||
from backend.audit import log_file_delete, log_file_save
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.history import (
|
||||
remove_recent,
|
||||
update_bookmarks_after_rename,
|
||||
update_history_after_rename,
|
||||
)
|
||||
from backend.indexer import handle_file_move, remove_single_file, update_single_file
|
||||
from backend.schemas import (
|
||||
BatchUploadRequest,
|
||||
BatchUploadResponse,
|
||||
DirectoryCreateRequest,
|
||||
DirectoryCreateResponse,
|
||||
DirectoryDeleteResponse,
|
||||
DirectoryRenameRequest,
|
||||
DirectoryRenameResponse,
|
||||
FileCreateRequest,
|
||||
FileCreateResponse,
|
||||
FileDeleteResponse,
|
||||
FileDuplicateRequest,
|
||||
FileDuplicateResponse,
|
||||
FileMoveRequest,
|
||||
FileMoveResponse,
|
||||
FileRenameRequest,
|
||||
FileRenameResponse,
|
||||
FileSaveResponse,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
batch_upload_files as service_batch_upload_files,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
create_directory as service_create_directory,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
create_file as service_create_file,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
delete_directory as service_delete_directory,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
delete_file as service_delete_file,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
duplicate_path as service_duplicate_path,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
edit_file as service_edit_file,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
edit_xlsx_cells as service_edit_xlsx_cells,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
move_path as service_move_path,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
mutate_xlsx_structure as service_mutate_xlsx_structure,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
mutate_xlsx_style as service_mutate_xlsx_style,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
rename_directory as service_rename_directory,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
rename_file as service_rename_file,
|
||||
)
|
||||
from backend.services.mutations import (
|
||||
save_csv_cells as service_save_csv_cells,
|
||||
)
|
||||
from backend.share import update_shares_after_rename
|
||||
from backend.sse import sse_manager
|
||||
from backend.webhooks import dispatch_webhooks
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["files"])
|
||||
|
||||
|
||||
@router.put("/api/file/{vault_name}/save", response_model=FileSaveResponse)
|
||||
async def api_file_save(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to file"),
|
||||
body: dict = Body(...),
|
||||
backup: bool = Query(True, description="Create a backup before saving (default true, set false for auto-save)"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Save (overwrite) a file's content.
|
||||
|
||||
Expects a JSON body with a ``content`` key containing the new text.
|
||||
The path is validated against traversal attacks before writing.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
body: JSON body with ``content`` string.
|
||||
|
||||
Returns:
|
||||
``FileSaveResponse`` confirming the write.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
content = body.get("content", "")
|
||||
result = service_edit_file(vault_name, path, content, backup=backup)
|
||||
|
||||
# #197 (synchro TikTak) : rafraîchir l'index de façon synchrone pour une
|
||||
# cohérence read-after-write — un client qui liste juste après le save
|
||||
# doit voir le nouveau mtime/size sans attendre le watcher.
|
||||
await update_single_file(vault_name, result["path"])
|
||||
|
||||
# #198 : la sauvegarde texte émet file_modified (événement clé pour tirer
|
||||
# les retours de synchro côté TikTak en phase 3).
|
||||
await dispatch_webhooks("file_modified", {"vault": vault_name, "path": result["path"]})
|
||||
|
||||
# Audit log
|
||||
client_ip = current_user.get("_request_ip", "unknown")
|
||||
log_file_save(current_user["username"], vault_name, path, len(content), client_ip)
|
||||
|
||||
# Journal du service d'agents (#220) — émission « au mieux », API seulement.
|
||||
agent_journal.emettre(agent_journal.TYPE_EDITE, vault_name, result["path"])
|
||||
|
||||
return {"status": "ok", "vault": result["vault"], "path": result["path"], "size": result["size"]}
|
||||
|
||||
|
||||
@router.put("/api/file/{vault_name}/xlsx/save", response_model=FileSaveResponse)
|
||||
def api_file_xlsx_save(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .xlsx file"),
|
||||
body: dict = Body(
|
||||
...,
|
||||
description=(
|
||||
'{"sheet": str, "cells": {"A1": value}, '
|
||||
'"allow_formula": false, "force": false, "if_match": str}'
|
||||
),
|
||||
),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Apply cell edits to an .xlsx workbook.
|
||||
|
||||
Expects a JSON body with ``sheet`` and ``cells`` (A1 references to new
|
||||
scalar values, max 500 per request) plus two optional boolean flags:
|
||||
|
||||
* ``allow_formula`` — keep values starting with ``=``/``@`` as real
|
||||
formulas. Off by default (#153 A4): such a value is stored as text so a
|
||||
later Excel session cannot execute it (DDE).
|
||||
* ``force`` — write a workbook carrying features openpyxl cannot re-serialize
|
||||
(slicers, form controls, connections, custom XML, signature, cached formula
|
||||
results). Without it the call fails **409** ``xlsx_lossy_content`` and the
|
||||
client asks the user to confirm (#153 A1).
|
||||
* ``if_match`` — revision token returned by the read (#156-A12). When it no
|
||||
longer matches the file on disk the write is refused with **409**
|
||||
``conflict`` (``reason=stale_revision``) instead of overwriting a change
|
||||
made by another writer. Omitted: last writer wins (curl, AI tools).
|
||||
|
||||
A backup is created before the workbook is rewritten, and the new archive
|
||||
swaps in atomically. Declared as a sync endpoint on purpose: the openpyxl
|
||||
round-trip and the per-file lock wait (#153 A3) then run in the threadpool
|
||||
instead of blocking the event loop.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
sheet = body.get("sheet")
|
||||
cells = body.get("cells")
|
||||
if not isinstance(sheet, str) or not sheet:
|
||||
raise HTTPException(status_code=400, detail="Feuille manquante")
|
||||
if not isinstance(cells, dict) or not cells or len(cells) > 500:
|
||||
raise HTTPException(status_code=400, detail="Cellules invalides (1 à 500 par requête)")
|
||||
for ref, value in cells.items():
|
||||
if not isinstance(ref, str) or not isinstance(value, (str, int, float, bool, type(None))):
|
||||
raise HTTPException(status_code=400, detail=f"Cellule invalide: {ref!r}")
|
||||
flags: dict[str, bool] = {}
|
||||
for name in ("allow_formula", "force"):
|
||||
raw = body.get(name, False)
|
||||
if not isinstance(raw, bool):
|
||||
raise HTTPException(status_code=400, detail=f"Flag invalide: {name}")
|
||||
flags[name] = raw
|
||||
if_match = body.get("if_match")
|
||||
if if_match is not None and not isinstance(if_match, str):
|
||||
raise HTTPException(status_code=400, detail="Jeton invalide: if_match")
|
||||
|
||||
result = service_edit_xlsx_cells(
|
||||
vault_name, path, sheet, cells, expected_revision=if_match, **flags
|
||||
)
|
||||
log_file_save(
|
||||
current_user["username"], vault_name, path,
|
||||
sum(len(str(v)) for v in cells.values()),
|
||||
current_user.get("_request_ip", "unknown"),
|
||||
)
|
||||
agent_journal.emettre(agent_journal.TYPE_EDITE, vault_name, result["path"])
|
||||
return {
|
||||
"status": "ok", "vault": result["vault"], "path": result["path"],
|
||||
"size": result["size"], "revision": result.get("revision"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/api/file/{vault_name}/csv/save", response_model=FileSaveResponse)
|
||||
def api_file_csv_save(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .csv file"),
|
||||
body: dict = Body(
|
||||
...,
|
||||
description=(
|
||||
'{"cells": {"A1": value}, "if_match": str} — A1-addressed text '
|
||||
'edits (#153 A16). `if_match` is the revision token of the read '
|
||||
'(#156-A12): a stale token fails with 409 `conflict`.'
|
||||
),
|
||||
),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Apply A1-addressed cell edits to a ``.csv`` file (#153 A16).
|
||||
|
||||
The grid is re-parsed with :mod:`csv`, patched and re-serialized
|
||||
(RFC 4180 quoting). References beyond the extent grow the grid. Values
|
||||
are stored verbatim as text — a CSV has no formula engine.
|
||||
|
||||
``if_match`` (optional, #156-A12) is the revision the client read: when the
|
||||
file changed in the meantime the write is refused with **409** ``conflict``.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
cells = body.get("cells")
|
||||
if not isinstance(cells, dict) or not cells or len(cells) > 500:
|
||||
raise HTTPException(status_code=400, detail="Cellules invalides (1 à 500 par requête)")
|
||||
for ref, value in cells.items():
|
||||
if not isinstance(ref, str) or not isinstance(value, (str, int, float, bool, type(None))):
|
||||
raise HTTPException(status_code=400, detail=f"Cellule invalide: {ref!r}")
|
||||
|
||||
if_match = body.get("if_match")
|
||||
if if_match is not None and not isinstance(if_match, str):
|
||||
raise HTTPException(status_code=400, detail="Jeton invalide: if_match")
|
||||
|
||||
result = service_save_csv_cells(vault_name, path, cells, expected_revision=if_match)
|
||||
log_file_save(
|
||||
current_user["username"], vault_name, path,
|
||||
sum(len(str(v)) for v in cells.values()),
|
||||
current_user.get("_request_ip", "unknown"),
|
||||
)
|
||||
agent_journal.emettre(agent_journal.TYPE_EDITE, vault_name, result["path"])
|
||||
return {
|
||||
"status": "ok", "vault": result["vault"], "path": result["path"],
|
||||
"size": result["size"], "revision": result.get("revision"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/api/file/{vault_name}/xlsx/structure", response_model=FileSaveResponse)
|
||||
def api_file_xlsx_structure(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .xlsx file"),
|
||||
body: dict = Body(
|
||||
...,
|
||||
description=(
|
||||
'{"actions": [{"op": "sheet_add", "name": "X"}, '
|
||||
'{"op": "row_insert", "sheet": "X", "at": 2, "count": 1}], '
|
||||
'"force": false, "if_match": str}'
|
||||
),
|
||||
),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Apply structural changes to an .xlsx workbook (#153 A14).
|
||||
|
||||
``actions`` is an ordered list applied in one locked, atomic rewrite:
|
||||
``sheet_add`` (``name``, optional ``at`` 0-based), ``sheet_rename``
|
||||
(``from``/``to``), ``sheet_delete`` (refused on the last sheet),
|
||||
``sheet_duplicate`` (``name``/``as``) and ``row_insert``/``row_delete``/
|
||||
``col_insert``/``col_delete`` (``sheet``, 1-based ``at``, ``count``).
|
||||
|
||||
Without ``force`` the call fails **409** ``xlsx_lossy_content`` when the
|
||||
workbook carries features openpyxl cannot rewrite (same gate as the cell
|
||||
edits). A backup is created before the archive is replaced. The optional
|
||||
``if_match`` revision (#156-A12) refuses a structural rewrite on a file that
|
||||
changed since it was read (**409** ``conflict``).
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path to the ``.xlsx`` file.
|
||||
body: JSON body with ``actions`` (1 to 50) and optional ``force``.
|
||||
|
||||
Returns:
|
||||
``FileSaveResponse`` confirming the write.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
actions = body.get("actions")
|
||||
if not isinstance(actions, list) or not actions or len(actions) > 50:
|
||||
raise HTTPException(status_code=400, detail="Actions invalides (1 à 50 par requête)")
|
||||
raw_force = body.get("force", False)
|
||||
if not isinstance(raw_force, bool):
|
||||
raise HTTPException(status_code=400, detail="Flag invalide: force")
|
||||
if_match = body.get("if_match")
|
||||
if if_match is not None and not isinstance(if_match, str):
|
||||
raise HTTPException(status_code=400, detail="Jeton invalide: if_match")
|
||||
|
||||
result = service_mutate_xlsx_structure(
|
||||
vault_name, path, actions, force=raw_force, expected_revision=if_match
|
||||
)
|
||||
log_file_save(
|
||||
current_user["username"], vault_name, path,
|
||||
len(actions),
|
||||
current_user.get("_request_ip", "unknown"),
|
||||
)
|
||||
agent_journal.emettre(agent_journal.TYPE_EDITE, vault_name, result["path"])
|
||||
return {
|
||||
"status": "ok", "vault": result["vault"], "path": result["path"],
|
||||
"size": len(result["applied"]), "revision": result.get("revision"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/api/file/{vault_name}/xlsx/style", response_model=FileSaveResponse)
|
||||
def api_file_xlsx_style(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to the .xlsx/.xlsm file"),
|
||||
body: dict = Body(
|
||||
...,
|
||||
description=(
|
||||
'{"ops": [{"op": "cell", "sheet": "X", "range": "A1:B2", '
|
||||
'"style": {"bold": true, "fill_color": "#ffe08a"}}], '
|
||||
'"force": false, "if_match": str}'
|
||||
),
|
||||
),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Write formatting on an .xlsx/.xlsm workbook (#156-A8).
|
||||
|
||||
``ops`` is an ordered list applied in one locked, atomic rewrite:
|
||||
``cell`` (``sheet``, ``range``/``cell``, ``style`` with ``bold``,
|
||||
``italic``, ``underline``, ``strike``, ``font_size`` (6-72),
|
||||
``font_color``/``fill_color`` as ``#rrggbb``, ``align``
|
||||
(left/center/right/justify), ``valign`` (top/middle/bottom), ``wrap``,
|
||||
``rotation`` (-90..90), ``border`` (all/outer/none) + ``border_style`` /
|
||||
``border_color`` and ``number_format``), ``merge``/``unmerge`` (``range``),
|
||||
``col_width`` (``col``, ``width``), ``row_height`` (``row``, ``height``),
|
||||
``comment_set`` (``ref``, ``text``) / ``comment_clear`` (``ref``)
|
||||
and ``freeze`` (``cell``, empty to release).
|
||||
|
||||
Without ``force`` the call fails **409** ``xlsx_lossy_content`` when the
|
||||
workbook carries features openpyxl cannot rewrite (same gate as the cell
|
||||
edits). A backup is created before the archive is replaced; the optional
|
||||
``if_match`` revision (#156-A12) refuses a rewrite on a file that changed
|
||||
since it was read (**409** ``conflict``).
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path to the ``.xlsx``/``.xlsm`` file.
|
||||
body: JSON body with ``ops`` (1 to 50) and optional ``force``.
|
||||
|
||||
Returns:
|
||||
``FileSaveResponse`` confirming the write.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
ops = body.get("ops")
|
||||
if not isinstance(ops, list) or not ops or len(ops) > 50:
|
||||
raise HTTPException(status_code=400, detail="Ops invalides (1 à 50 par requête)")
|
||||
raw_force = body.get("force", False)
|
||||
if not isinstance(raw_force, bool):
|
||||
raise HTTPException(status_code=400, detail="Flag invalide: force")
|
||||
if_match = body.get("if_match")
|
||||
if if_match is not None and not isinstance(if_match, str):
|
||||
raise HTTPException(status_code=400, detail="Jeton invalide: if_match")
|
||||
|
||||
result = service_mutate_xlsx_style(
|
||||
vault_name, path, ops, force=raw_force, expected_revision=if_match
|
||||
)
|
||||
log_file_save(
|
||||
current_user["username"], vault_name, path,
|
||||
len(ops),
|
||||
current_user.get("_request_ip", "unknown"),
|
||||
)
|
||||
agent_journal.emettre(agent_journal.TYPE_EDITE, vault_name, result["path"])
|
||||
return {
|
||||
"status": "ok", "vault": result["vault"], "path": result["path"],
|
||||
"size": len(result["applied"]), "revision": result.get("revision"),
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/api/file/{vault_name}", response_model=FileDeleteResponse)
|
||||
async def api_file_delete(vault_name: str, path: str = Query(..., description="Relative path to file"), current_user=Depends(require_auth)):
|
||||
"""Delete a file from the vault.
|
||||
|
||||
The path is validated against traversal attacks before deletion.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative file path within the vault.
|
||||
|
||||
Returns:
|
||||
``FileDeleteResponse`` confirming the deletion.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_delete_file(vault_name, path)
|
||||
|
||||
# Journal du service d'agents (#220) — émission « au mieux », API seulement.
|
||||
agent_journal.emettre(agent_journal.TYPE_SUPPRIME, vault_name, result["path"])
|
||||
|
||||
# Audit log
|
||||
client_ip = current_user.get("_request_ip", "unknown")
|
||||
log_file_delete(current_user["username"], vault_name, path, client_ip)
|
||||
|
||||
# Update index
|
||||
await remove_single_file(vault_name, path)
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("file_deleted", {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
})
|
||||
|
||||
from backend.plugins import emit_file_deleted
|
||||
emit_file_deleted(vault_name, path)
|
||||
|
||||
# Remove from recent files
|
||||
remove_recent(current_user["username"], vault_name, path)
|
||||
|
||||
# Dispatch webhooks
|
||||
await dispatch_webhooks("file_deleted", {"vault": vault_name, "path": path})
|
||||
|
||||
return {"status": "ok", "vault": result["vault"], "path": result["path"]}
|
||||
|
||||
|
||||
@router.post("/api/directory/{vault_name}", response_model=DirectoryCreateResponse)
|
||||
async def api_directory_create(
|
||||
vault_name: str,
|
||||
body: DirectoryCreateRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Create a new directory in a vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Request body with directory path.
|
||||
|
||||
Returns:
|
||||
DirectoryCreateResponse confirming creation.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_create_directory(vault_name, body.path)
|
||||
|
||||
# Update path_index with the new directory
|
||||
from backend.indexer import _index_lock
|
||||
from backend.indexer import path_index as _path_idx
|
||||
with _index_lock:
|
||||
if vault_name not in _path_idx:
|
||||
_path_idx[vault_name] = []
|
||||
existing = {p["path"] for p in _path_idx[vault_name]}
|
||||
# Build all parent segments
|
||||
parts = body.path.split("/")
|
||||
for i in range(1, len(parts) + 1):
|
||||
seg_path = "/".join(parts[:i])
|
||||
if seg_path and seg_path not in existing:
|
||||
existing.add(seg_path)
|
||||
_path_idx[vault_name].append({
|
||||
"path": seg_path,
|
||||
"name": parts[i - 1],
|
||||
"type": "directory",
|
||||
})
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("directory_created", {
|
||||
"vault": vault_name,
|
||||
"path": result["path"],
|
||||
})
|
||||
await dispatch_webhooks("directory_created", {"vault": vault_name, "path": result["path"]})
|
||||
|
||||
return {"success": True, "path": result["path"]}
|
||||
|
||||
|
||||
@router.patch("/api/directory/{vault_name}", response_model=DirectoryRenameResponse)
|
||||
async def api_directory_rename(
|
||||
vault_name: str,
|
||||
body: DirectoryRenameRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Rename a directory in a vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Request body with current path and new name.
|
||||
|
||||
Returns:
|
||||
DirectoryRenameResponse with old and new paths.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_rename_directory(vault_name, body.path, body.new_name)
|
||||
old_path_str = result["old_path"]
|
||||
new_path_str = result["new_path"]
|
||||
|
||||
# Update index for all files in the directory
|
||||
from backend.indexer import reload_single_vault
|
||||
await reload_single_vault(vault_name)
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("directory_renamed", {
|
||||
"vault": vault_name,
|
||||
"old_path": old_path_str,
|
||||
"new_path": new_path_str,
|
||||
})
|
||||
await dispatch_webhooks("directory_renamed", {"vault": vault_name, "old_path": old_path_str, "new_path": new_path_str})
|
||||
|
||||
return {"success": True, "old_path": old_path_str, "new_path": new_path_str}
|
||||
|
||||
|
||||
@router.delete("/api/directory/{vault_name}", response_model=DirectoryDeleteResponse)
|
||||
async def api_directory_delete(
|
||||
vault_name: str,
|
||||
path: str = Query(..., description="Relative path to directory"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Delete a directory and all its contents from a vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative directory path within the vault.
|
||||
|
||||
Returns:
|
||||
DirectoryDeleteResponse with count of deleted files.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_delete_directory(vault_name, path, recursive=True)
|
||||
file_count = result["deleted_count"]
|
||||
|
||||
# Update index — incremental subtree removal (no full vault re-scan, no
|
||||
# global inverted-index rebuild).
|
||||
from backend.indexer import remove_directory_from_index
|
||||
await remove_directory_from_index(vault_name, result["path"])
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("directory_deleted", {
|
||||
"vault": vault_name,
|
||||
"path": result["path"],
|
||||
"deleted_count": file_count,
|
||||
})
|
||||
await dispatch_webhooks("directory_deleted", {"vault": vault_name, "path": result["path"]})
|
||||
|
||||
return {"success": True, "deleted_count": file_count}
|
||||
|
||||
|
||||
@router.post("/api/file/{vault_name}", response_model=FileCreateResponse)
|
||||
async def api_file_create(
|
||||
vault_name: str,
|
||||
body: FileCreateRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Create a new file in a vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Request body with file path and initial content.
|
||||
|
||||
Returns:
|
||||
FileCreateResponse confirming creation.
|
||||
|
||||
Note:
|
||||
A ``.xlsx`` path creates an empty workbook (one ``Feuille1`` sheet) built
|
||||
with openpyxl: the payload is binary, so ``content`` is ignored (#186).
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_create_file(vault_name, body.path, body.content)
|
||||
|
||||
# Journal du service d'agents (#220) — émission « au mieux », API seulement.
|
||||
agent_journal.emettre(agent_journal.TYPE_CREE, vault_name, result["path"])
|
||||
|
||||
# Update index
|
||||
await update_single_file(vault_name, result["path"])
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("file_created", {
|
||||
"vault": vault_name,
|
||||
"path": result["path"],
|
||||
})
|
||||
await dispatch_webhooks("file_created", {"vault": vault_name, "path": result["path"]})
|
||||
from backend.plugins import emit_file_created
|
||||
emit_file_created(vault_name, result["path"])
|
||||
|
||||
return {"success": True, "path": result["path"]}
|
||||
|
||||
|
||||
@router.post("/api/vault/{vault_name}/batch-upload", response_model=BatchUploadResponse)
|
||||
async def api_batch_upload(
|
||||
vault_name: str,
|
||||
body: BatchUploadRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Upload multiple files and directories (recursively) into a vault.
|
||||
|
||||
Accepts base64 encoded or plain text files with relative directory paths.
|
||||
Creates missing parent folders safely.
|
||||
|
||||
Args:
|
||||
vault_name: Target vault name.
|
||||
body: BatchUploadRequest with target_dir and files list.
|
||||
|
||||
Returns:
|
||||
BatchUploadResponse with summary of uploaded files and errors.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
import base64
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
for f in body.files:
|
||||
if f.is_dir:
|
||||
items.append({"path": f.path, "is_dir": True})
|
||||
continue
|
||||
|
||||
raw_bytes = b""
|
||||
if f.content is not None:
|
||||
# Check if content is base64 encoded data URI or raw base64
|
||||
content_str = f.content
|
||||
if content_str.startswith("data:") and ";base64," in content_str:
|
||||
content_str = content_str.split(";base64,", 1)[1]
|
||||
try:
|
||||
raw_bytes = base64.b64decode(content_str)
|
||||
except Exception:
|
||||
# Fallback to utf-8 text encoding
|
||||
raw_bytes = f.content.encode("utf-8")
|
||||
|
||||
items.append({"path": f.path, "content": raw_bytes, "is_dir": False})
|
||||
|
||||
result = service_batch_upload_files(
|
||||
vault_name,
|
||||
body.target_dir,
|
||||
items,
|
||||
overwrite=body.overwrite,
|
||||
)
|
||||
|
||||
# Update index and SSE notifications for uploaded files
|
||||
for path in result["uploaded"]:
|
||||
try:
|
||||
await update_single_file(vault_name, path)
|
||||
await sse_manager.broadcast("file_created", {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
})
|
||||
await dispatch_webhooks("file_created", {"vault": vault_name, "path": path})
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to post-process upload of {path}: {e}")
|
||||
|
||||
# SSE notification for tree refresh
|
||||
if result["uploaded"] or result["created_dirs"]:
|
||||
await sse_manager.broadcast("tree_updated", {
|
||||
"vault": vault_name,
|
||||
"target_dir": result["target_dir"],
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.patch("/api/file/{vault_name}", response_model=FileRenameResponse)
|
||||
async def api_file_rename(
|
||||
vault_name: str,
|
||||
body: FileRenameRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Rename a file in a vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Request body with current path and new name.
|
||||
|
||||
Returns:
|
||||
FileRenameResponse with old and new paths.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_rename_file(vault_name, body.path, body.new_name)
|
||||
old_path_str = result["old_path"]
|
||||
new_path_str = result["new_path"]
|
||||
|
||||
# Journal du service d'agents (#220) — la référence pointe sur le nouveau chemin.
|
||||
agent_journal.emettre(agent_journal.TYPE_DEPLACE, vault_name, new_path_str)
|
||||
|
||||
# Update index
|
||||
await handle_file_move(vault_name, old_path_str, new_path_str)
|
||||
|
||||
# Update bookmarks, history, and shares
|
||||
update_bookmarks_after_rename(vault_name, old_path_str, new_path_str)
|
||||
update_history_after_rename(vault_name, old_path_str, new_path_str)
|
||||
update_shares_after_rename(vault_name, old_path_str, new_path_str)
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("file_renamed", {
|
||||
"vault": vault_name,
|
||||
"old_path": old_path_str,
|
||||
"new_path": new_path_str,
|
||||
})
|
||||
await dispatch_webhooks("file_renamed", {"vault": vault_name, "old_path": old_path_str, "new_path": new_path_str})
|
||||
|
||||
return {"success": True, "old_path": old_path_str, "new_path": new_path_str}
|
||||
|
||||
|
||||
@router.post("/api/move/{vault_name}", response_model=FileMoveResponse)
|
||||
async def api_file_move(
|
||||
vault_name: str,
|
||||
body: FileMoveRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Move a file or directory to a different parent directory within the same vault.
|
||||
|
||||
Supports both files and directories. The item keeps its original name;
|
||||
only the parent directory changes.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
body: Request body with source_path and destination_dir.
|
||||
|
||||
Returns:
|
||||
FileMoveResponse with old and new paths.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_move_path(vault_name, body.source_path, body.destination_dir)
|
||||
old_path_str = result["old_path"]
|
||||
new_path_str = result["new_path"]
|
||||
item_type = result["item_type"]
|
||||
|
||||
# Journal du service d'agents (#220) — fichiers seulement (le catalogue
|
||||
# est note.*) ; les déplacements de dossiers ne sont pas émis.
|
||||
if item_type == "file":
|
||||
agent_journal.emettre(agent_journal.TYPE_DEPLACE, vault_name, new_path_str)
|
||||
|
||||
# Update index
|
||||
if item_type == "directory":
|
||||
from backend.indexer import reload_single_vault
|
||||
await reload_single_vault(vault_name)
|
||||
else:
|
||||
await handle_file_move(vault_name, old_path_str, new_path_str)
|
||||
|
||||
# Broadcast SSE event
|
||||
await sse_manager.broadcast("item_moved", {
|
||||
"vault": vault_name,
|
||||
"old_path": old_path_str,
|
||||
"new_path": new_path_str,
|
||||
"item_type": item_type,
|
||||
})
|
||||
await dispatch_webhooks("item_moved", {"vault": vault_name, "old_path": old_path_str, "new_path": new_path_str, "item_type": item_type})
|
||||
|
||||
return {"success": True, "old_path": old_path_str, "new_path": new_path_str, "item_type": item_type}
|
||||
|
||||
@router.post("/api/duplicate/{vault_name}", response_model=FileDuplicateResponse)
|
||||
async def api_file_duplicate(
|
||||
vault_name: str,
|
||||
body: FileDuplicateRequest,
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Duplicate (copy) a file or directory within the same vault."""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
result = service_duplicate_path(vault_name, body.source_path, body.destination_dir)
|
||||
new_path_str = result["new_path"]
|
||||
item_type = result["item_type"]
|
||||
|
||||
# Index: incrémental pour chaque fichier copié (évite un reload complet).
|
||||
from backend.services.vaults import get_vault_root
|
||||
root = get_vault_root(vault_name)
|
||||
target = root / new_path_str
|
||||
if item_type == "directory":
|
||||
for fp in sorted(target.rglob("*")):
|
||||
if fp.is_file():
|
||||
await update_single_file(vault_name, str(fp.relative_to(root)).replace("\\", "/"))
|
||||
else:
|
||||
await update_single_file(vault_name, new_path_str)
|
||||
|
||||
await sse_manager.broadcast("item_duplicated", {
|
||||
"vault": vault_name,
|
||||
"source_path": result["source_path"],
|
||||
"new_path": new_path_str,
|
||||
"item_type": item_type,
|
||||
})
|
||||
await dispatch_webhooks(
|
||||
"file_created" if item_type == "file" else "directory_created",
|
||||
{"vault": vault_name, "path": new_path_str},
|
||||
)
|
||||
|
||||
return {"success": True, "source_path": result["source_path"], "new_path": new_path_str, "item_type": item_type}
|
||||
@@ -0,0 +1,143 @@
|
||||
"""System health endpoints (ROADMAP #85, tranche 1).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/health``, ``/api/health/detailed``),
|
||||
même ``response_model`` (:class:`backend.schemas.HealthResponse`), même
|
||||
dépendance admin. Seule différence : la version est lue via
|
||||
:func:`backend.version.get_version` au lieu de ``app.version`` (valeur
|
||||
identique, figée au démarrage depuis le fichier ``VERSION``).
|
||||
|
||||
Note : ``uptime_seconds`` reprend l'expression d'origine
|
||||
(``'_SERVER_START_TIME' in globals()``), qui vaut toujours 0 — le global
|
||||
n'est défini nulle part dans ``backend.main`` (voir ``backend.admin`` qui
|
||||
possède son propre compteur). Ce comportement est préservé tel quel ; le
|
||||
corriger fera l'objet d'une tranche ultérieure avec test dédié.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from backend.auth.middleware import require_admin
|
||||
from backend.indexer import index
|
||||
from backend.schemas import HealthResponse
|
||||
from backend.version import get_git_commit, get_git_describe, get_version
|
||||
|
||||
router = APIRouter(tags=["System"])
|
||||
|
||||
|
||||
@router.get("/api/health", response_model=HealthResponse)
|
||||
async def api_health():
|
||||
"""Health check endpoint for Docker and monitoring.
|
||||
|
||||
Returns:
|
||||
Application status, version, vault count and total file count.
|
||||
"""
|
||||
total_files = sum(len(v["files"]) for v in index.values())
|
||||
total_tokens = sum(len(v.get("files", [])) * 1000 for v in index.values()) # rough approx
|
||||
import time
|
||||
|
||||
from backend.indexer import _last_full_index_ts
|
||||
# `_SERVER_START_TIME` n'existe dans aucun module (comportement d'origine
|
||||
# préservé : uptime toujours 0 — voir docstring du module).
|
||||
uptime = int(time.time() - _SERVER_START_TIME) if '_SERVER_START_TIME' in globals() else 0 # noqa: F821
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": get_version(),
|
||||
"vaults": len(index),
|
||||
"total_files": total_files,
|
||||
"total_tokens": total_tokens,
|
||||
"last_full_index_ts": _last_full_index_ts,
|
||||
"uptime_seconds": uptime,
|
||||
"git_describe": get_git_describe(),
|
||||
"git_commit": get_git_commit(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/health/detailed", response_model=HealthResponse)
|
||||
async def api_health_detailed(current_user=Depends(require_admin)):
|
||||
"""Detailed health check — admin only.
|
||||
|
||||
Returns enriched metrics including memory, disk, SSE connections, and backup stats.
|
||||
"""
|
||||
|
||||
import psutil
|
||||
|
||||
from backend.admin import _count_active_sessions, _get_disk_stats
|
||||
from backend.indexer import _last_full_index_ts, index
|
||||
|
||||
total_files = sum(len(v["files"]) for v in index.values())
|
||||
total_tokens = sum(len(v.get("files", [])) * 1000 for v in index.values())
|
||||
import time
|
||||
uptime = int(time.time() - _SERVER_START_TIME) if '_SERVER_START_TIME' in globals() else 0 # noqa: F821 — voir ci-dessus
|
||||
|
||||
# Memory
|
||||
vm = psutil.virtual_memory()
|
||||
mem_used_mb = round(vm.used / (1024 ** 2), 1)
|
||||
mem_total_mb = round(vm.total / (1024 ** 2), 1)
|
||||
mem_pct = round(vm.percent, 1)
|
||||
|
||||
# CPU
|
||||
cpu_pct = psutil.cpu_percent(interval=None)
|
||||
|
||||
# Disk
|
||||
disk_used_gb, disk_total_gb = _get_disk_stats()
|
||||
disk_free_gb = round(disk_total_gb - disk_used_gb, 2)
|
||||
disk_pct = round((disk_used_gb / disk_total_gb * 100) if disk_total_gb > 0 else 0, 1)
|
||||
|
||||
# SSE connections (approximation)
|
||||
active_sessions = _count_active_sessions()
|
||||
|
||||
# Backups
|
||||
from backend.admin import _scan_backups
|
||||
backup_rows = _scan_backups()
|
||||
total_backups = len(backup_rows)
|
||||
total_backup_size_mb = round(sum(r["size"] for r in backup_rows) / (1024 ** 2), 2)
|
||||
oldest_backup_age_days = 0.0
|
||||
if backup_rows:
|
||||
now_ts = int(time.time())
|
||||
oldest_ts = min(r["timestamp"] for r in backup_rows)
|
||||
oldest_backup_age_days = round((now_ts - oldest_ts) / 86400, 2)
|
||||
|
||||
# Index details
|
||||
index_detail = {}
|
||||
for name, data in index.items():
|
||||
index_detail[name] = {
|
||||
"file_count": len(data["files"]),
|
||||
"tag_count": len(data.get("tags", [])),
|
||||
"token_count_approx": len(data.get("files", [])) * 1000,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": get_version(),
|
||||
"vaults": len(index),
|
||||
"total_files": total_files,
|
||||
"total_tokens": total_tokens,
|
||||
"last_full_index_ts": _last_full_index_ts,
|
||||
"uptime_seconds": uptime,
|
||||
"git_describe": get_git_describe(),
|
||||
"git_commit": get_git_commit(),
|
||||
# Enriched fields
|
||||
"memory": {
|
||||
"used_mb": mem_used_mb,
|
||||
"total_mb": mem_total_mb,
|
||||
"percent": mem_pct,
|
||||
},
|
||||
"cpu": {
|
||||
"percent": cpu_pct,
|
||||
},
|
||||
"disk": {
|
||||
"used_gb": disk_used_gb,
|
||||
"total_gb": disk_total_gb,
|
||||
"free_gb": disk_free_gb,
|
||||
"percent": disk_pct,
|
||||
},
|
||||
"connections": {
|
||||
"active_sse": active_sessions,
|
||||
},
|
||||
"backups": {
|
||||
"total_count": total_backups,
|
||||
"total_size_mb": total_backup_size_mb,
|
||||
"oldest_age_days": oldest_backup_age_days,
|
||||
},
|
||||
"index": index_detail,
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Shared helpers for the file routers (ROADMAP #85, tranche 6a).
|
||||
|
||||
Petites fonctions pures extraites de :mod:`backend.main` sans changement
|
||||
de comportement. Regroupées ici car utilisées par plusieurs routers
|
||||
(``files_read`` aujourd'hui, ``files_media`` / mutations ensuite) :
|
||||
- :func:`content_disposition` — aussi utilisée par ``_stream_file_with_range``
|
||||
(resté dans ``main`` jusqu'à la tranche media).
|
||||
- :func:`media_max_inline_bytes` — aussi utilisée par ``/api/media``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
|
||||
|
||||
def content_disposition(disposition: str, filename: str) -> str:
|
||||
"""Build a header-safe Content-Disposition value.
|
||||
|
||||
HTTP header values must be ASCII. Unicode filenames are sent per
|
||||
RFC 5987 via ``filename*`` (percent-encoded UTF-8) with a pure-ASCII
|
||||
``filename`` fallback. This avoids a UnicodeDecodeError / HTTP 500 when
|
||||
the filename contains accented characters (e.g. 'Bière blonde…pdf').
|
||||
"""
|
||||
from urllib.parse import quote
|
||||
ascii_name = "".join(c for c in filename if c.isascii() and (c.isalnum() or c in " _-.")).strip() or "file"
|
||||
ext = Path(filename).suffix
|
||||
if ext and not Path(ascii_name).suffix:
|
||||
ascii_name = ascii_name + ext
|
||||
return f"{disposition}; filename=\"{ascii_name}\"; filename*=UTF-8''{quote(filename)}"
|
||||
|
||||
|
||||
def media_max_inline_bytes() -> int:
|
||||
"""Maximum size (bytes) for inline audio/video playback (roadmap #109-A3).
|
||||
|
||||
Configurable via ``OBSIGATE_MEDIA_MAX_INLINE_MB`` (default 500 MB). Files
|
||||
above the limit are not streamed in the viewer (the UI falls back to the
|
||||
download button), which keeps a single uvicorn worker from being pinned by
|
||||
multi-gigabyte media. Invalid or non-positive values fall back to default.
|
||||
"""
|
||||
default_mb = 500
|
||||
raw = os.environ.get("OBSIGATE_MEDIA_MAX_INLINE_MB", "").strip()
|
||||
if not raw:
|
||||
return default_mb * 1024 * 1024
|
||||
try:
|
||||
mb = float(raw)
|
||||
except ValueError:
|
||||
return default_mb * 1024 * 1024
|
||||
if mb <= 0:
|
||||
return default_mb * 1024 * 1024
|
||||
return int(mb * 1024 * 1024)
|
||||
|
||||
|
||||
def stream_file_with_range(file_path: Path, request: Request, media_type: str):
|
||||
"""Return a file response honouring the HTTP ``Range`` header (roadmap #109).
|
||||
|
||||
Extrait de :mod:`backend.main` (``_stream_file_with_range``) sans
|
||||
changement de comportement. Shared by ``pdf/stream`` and ``/api/media``:
|
||||
a plain :class:`FileResponse` with ``Accept-Ranges: bytes`` when no range
|
||||
is requested, or a :class:`StreamingResponse` (206 Partial Content,
|
||||
64 KiB chunks) for a valid single range. An unsatisfiable range yields
|
||||
``416`` with a ``Content-Range: bytes */size`` header.
|
||||
|
||||
Reads are offloaded to threads so the event loop is never blocked
|
||||
(ASYNC230), matching the previous inline implementation.
|
||||
"""
|
||||
file_size = file_path.stat().st_size
|
||||
range_header = request.headers.get("range")
|
||||
disposition = content_disposition("inline", file_path.name)
|
||||
|
||||
if range_header:
|
||||
# Parse "bytes=start-end" (single range only; multi-range is not used by viewers)
|
||||
m = re.match(r"bytes=(\d*)-(\d*)", range_header)
|
||||
if not m:
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
start_s, end_s = m.group(1), m.group(2)
|
||||
if start_s == "" and end_s == "":
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
if start_s == "":
|
||||
# suffix range: last N bytes
|
||||
length = min(int(end_s), file_size)
|
||||
start = file_size - length
|
||||
end = file_size - 1
|
||||
else:
|
||||
start = int(start_s)
|
||||
end = int(end_s) if end_s else file_size - 1
|
||||
end = min(end, file_size - 1)
|
||||
if start > end or start >= file_size:
|
||||
raise HTTPException(status_code=416,
|
||||
headers={"Content-Range": f"bytes */{file_size}"})
|
||||
|
||||
chunk_size = end - start + 1
|
||||
|
||||
async def _partial():
|
||||
f = await asyncio.to_thread(open, str(file_path), "rb")
|
||||
try:
|
||||
await asyncio.to_thread(f.seek, start)
|
||||
remaining = chunk_size
|
||||
while remaining > 0:
|
||||
data = await asyncio.to_thread(f.read, min(64 * 1024, remaining))
|
||||
if not data:
|
||||
break
|
||||
remaining -= len(data)
|
||||
yield data
|
||||
finally:
|
||||
await asyncio.to_thread(f.close)
|
||||
|
||||
return StreamingResponse(
|
||||
_partial(),
|
||||
status_code=206,
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Content-Range": f"bytes {start}-{end}/{file_size}",
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Length": str(chunk_size),
|
||||
"Content-Disposition": disposition,
|
||||
},
|
||||
)
|
||||
|
||||
return FileResponse(str(file_path), media_type=media_type, headers={
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Disposition": disposition})
|
||||
@@ -0,0 +1,160 @@
|
||||
"""History endpoints — recent, bookmarks, saved searches (ROADMAP #85, tranche 8).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins, mêmes modèles (``BookmarkToggleRequest``
|
||||
déménagé dans :mod:`backend.schemas`), mêmes dépendances
|
||||
d'authentification.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- ``_resolve_safe_path`` / ``_backup_file`` → :mod:`backend.services.paths`
|
||||
et :mod:`backend.services.backups` (pass-through).
|
||||
- ``_load_config`` vient de :mod:`backend.routers.config`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import frontmatter
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.history import get_bookmarks, toggle_bookmark
|
||||
from backend.indexer import find_file_in_index, get_vault_data, update_single_file
|
||||
from backend.routers.config import _load_config
|
||||
from backend.saved_searches import delete_saved, get_saved, save_search
|
||||
from backend.schemas import (
|
||||
BookmarksResponse,
|
||||
BookmarkToggleRequest,
|
||||
BookmarkToggleResponse,
|
||||
RecentResponse,
|
||||
SavedSearch,
|
||||
StatusResponse,
|
||||
)
|
||||
from backend.services.backups import create_backup
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.recent import humanize_mtime, list_recent
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["Bookmarks"])
|
||||
|
||||
|
||||
@router.get("/api/recent", response_model=RecentResponse)
|
||||
async def api_recent(limit: int | None = Query(None), vault: str | None = Query(None), mode: str | None = Query("opened"), current_user=Depends(require_auth)):
|
||||
config = _load_config()
|
||||
actual_limit = limit if limit is not None else config.get("recent_files_limit", 20)
|
||||
|
||||
username = current_user.get("username")
|
||||
user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", [])
|
||||
|
||||
return list_recent(
|
||||
username,
|
||||
user_vaults,
|
||||
vault=vault,
|
||||
limit=actual_limit,
|
||||
mode=mode or "opened",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/bookmarks", response_model=BookmarksResponse)
|
||||
async def api_bookmarks(vault: str | None = Query(None), current_user=Depends(require_auth)):
|
||||
username = current_user.get("username")
|
||||
|
||||
if not username:
|
||||
return {"files": []}
|
||||
|
||||
history = get_bookmarks(username, vault_filter=vault)
|
||||
files_resp = []
|
||||
for item in history:
|
||||
v_name = item["vault"]
|
||||
# #194 : check_vault_access — "*" n'inclut pas les dossiers persos.
|
||||
if not check_vault_access(v_name, current_user):
|
||||
continue
|
||||
|
||||
# Find in index to get metadata
|
||||
f_idx = find_file_in_index(item["path"], v_name)
|
||||
if f_idx:
|
||||
files_resp.append({
|
||||
"path": f_idx["path"],
|
||||
"title": f_idx.get("title") or item["path"].split("/")[-1],
|
||||
"vault": v_name,
|
||||
"mtime": item["bookmarked_at"],
|
||||
"mtime_human": humanize_mtime(item["bookmarked_at"]),
|
||||
"size_bytes": f_idx.get("size", 0),
|
||||
"tags": [f"#{t}" for t in f_idx.get("tags", [])][:5],
|
||||
"bookmarked": True
|
||||
})
|
||||
else:
|
||||
files_resp.append({
|
||||
"path": item["path"],
|
||||
"title": item.get("title") or item["path"].split("/")[-1],
|
||||
"vault": v_name,
|
||||
"mtime": item["bookmarked_at"],
|
||||
"mtime_human": humanize_mtime(item["bookmarked_at"]),
|
||||
"tags": [],
|
||||
"bookmarked": True
|
||||
})
|
||||
return {
|
||||
"files": files_resp,
|
||||
"total": len(files_resp)
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/bookmarks/toggle", response_model=BookmarkToggleResponse)
|
||||
async def api_toggle_bookmark(req: BookmarkToggleRequest, current_user=Depends(require_auth)):
|
||||
username = current_user.get("username")
|
||||
if not username:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
|
||||
# Check vault access
|
||||
if not check_vault_access(req.vault, current_user):
|
||||
raise HTTPException(status_code=403, detail="Access denied to vault")
|
||||
|
||||
is_now_bookmarked = toggle_bookmark(username, req.vault, req.path, req.title or "")
|
||||
|
||||
# Update the file's YAML frontmatter: favoris: true/false
|
||||
vault_data = get_vault_data(req.vault)
|
||||
if vault_data:
|
||||
file_path = resolve_safe_path(Path(vault_data["path"]), req.path)
|
||||
if file_path.exists() and file_path.suffix == ".md":
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
post = frontmatter.loads(raw)
|
||||
if is_now_bookmarked:
|
||||
post.metadata["favoris"] = True
|
||||
elif "favoris" in post.metadata:
|
||||
del post.metadata["favoris"]
|
||||
new_raw = frontmatter.dumps(post)
|
||||
create_backup(file_path, req.vault, req.path)
|
||||
file_path.write_text(new_raw, encoding="utf-8")
|
||||
await update_single_file(req.vault, str(file_path))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update favoris metadata on {req.vault}/{req.path}: {e}")
|
||||
|
||||
return {"bookmarked": is_now_bookmarked}
|
||||
|
||||
|
||||
@router.get("/api/saved-searches", response_model=list[SavedSearch])
|
||||
async def api_saved_searches(current_user=Depends(require_auth)):
|
||||
username = current_user.get("username")
|
||||
if not username:
|
||||
raise HTTPException(401)
|
||||
return get_saved(username)
|
||||
|
||||
|
||||
@router.post("/api/saved-searches", response_model=SavedSearch)
|
||||
async def api_save_search(body: dict = Body(...), current_user=Depends(require_auth)):
|
||||
username = current_user.get("username")
|
||||
if not username:
|
||||
raise HTTPException(401)
|
||||
return save_search(username, body)
|
||||
|
||||
|
||||
@router.delete("/api/saved-searches/{search_id}", response_model=StatusResponse)
|
||||
async def api_delete_saved_search(search_id: str, current_user=Depends(require_auth)):
|
||||
username = current_user.get("username")
|
||||
if not username:
|
||||
raise HTTPException(401)
|
||||
if not delete_saved(username, search_id):
|
||||
raise HTTPException(404, "Not found")
|
||||
return {"status": "deleted"}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""External notification channels endpoints (#168).
|
||||
|
||||
Channel CRUD is admin-only (secrets involved); sending a test notification
|
||||
requires authentication. Responses mask secrets (``***`` + ``has_secret``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend import notify as _notify
|
||||
from backend.auth.middleware import require_admin, require_auth
|
||||
from backend.schemas import StatusResponse
|
||||
|
||||
router = APIRouter(prefix="/api/notify", tags=["notify"])
|
||||
|
||||
|
||||
class NotifyChannel(BaseModel):
|
||||
"""Public view of a notification channel (secrets masked)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
id: str = Field(description="Channel id")
|
||||
name: str = Field(description="Display name")
|
||||
type: str = Field(description="discord | telegram | smtp | webhook")
|
||||
enabled: bool = Field(description="Whether the channel receives broadcasts")
|
||||
config: dict[str, Any] = Field(description="Channel config (secrets masked)")
|
||||
has_secret: bool = Field(description="True when a secret is configured")
|
||||
|
||||
|
||||
class NotifySendResult(BaseModel):
|
||||
"""Outcome of a test send / broadcast."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
ok: bool = Field(description="True when every delivery succeeded")
|
||||
deliveries: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/channels", response_model=list[NotifyChannel])
|
||||
async def api_notify_list(current_user=Depends(require_admin)):
|
||||
"""List notification channels (admin)."""
|
||||
return _notify.list_channels()
|
||||
|
||||
|
||||
@router.post("/channels", response_model=NotifyChannel)
|
||||
async def api_notify_create(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Create a channel (``{name, type, config}``). Secrets go to the secret store."""
|
||||
try:
|
||||
return _notify.create_channel(
|
||||
str(body.get("name") or ""),
|
||||
str(body.get("type") or ""),
|
||||
dict(body.get("config") or {}),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/channels/{channel_id}", response_model=NotifyChannel)
|
||||
async def api_notify_update(channel_id: str, body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Update a channel (name / enabled / config)."""
|
||||
try:
|
||||
result = _notify.update_channel(channel_id, body)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if result is None:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/channels/{channel_id}", response_model=StatusResponse)
|
||||
async def api_notify_delete(channel_id: str, current_user=Depends(require_admin)):
|
||||
"""Delete a channel and its secret."""
|
||||
if not _notify.delete_channel(channel_id):
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/test", response_model=NotifySendResult)
|
||||
async def api_notify_test(body: dict = Body(...), current_user=Depends(require_auth)):
|
||||
"""Send a test notification (broadcast or single ``channel_id``)."""
|
||||
title = str(body.get("title") or "Test ObsiGate")
|
||||
message = str(body.get("message") or "Notification de test.")
|
||||
channel_id = str(body.get("channel_id") or "")
|
||||
if channel_id:
|
||||
channel = next((c for c in _notify._read_channels() if c.get("id") == channel_id), None)
|
||||
if channel is None:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
try:
|
||||
_notify.send_via_channel(channel, title, message, "manual")
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Envoi échoué : {e}") from e
|
||||
return {"ok": True, "deliveries": [{"channel_id": channel_id, "ok": True}]}
|
||||
deliveries = _notify.broadcast("manual", title, message)
|
||||
return {"ok": all(d.get("ok") for d in deliveries), "deliveries": deliveries}
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Real-time endpoints — SSE stream & collaboration WebSocket (ROADMAP #85, tranche 9).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/events``,
|
||||
``/ws/collab/{vault}/{path}``), même authentification (Depend pour le SSE,
|
||||
manuelle pour le WebSocket — les ``Depends`` FastAPI ne s'exécutent pas sur
|
||||
les routes WebSocket).
|
||||
|
||||
Pas de tags déclarés : assignation par chemin via
|
||||
``openapi_docs.tag_for_path`` comme avant (``/api/events`` → System).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
|
||||
from fastapi import APIRouter, Depends, WebSocket
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.collab import authenticate_websocket, collab_manager
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import get_vault_root
|
||||
from backend.sse import sse_manager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/events",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"text/event-stream": {}}, "description": "Server-Sent Events stream"}},
|
||||
)
|
||||
async def api_events(current_user=Depends(require_auth)):
|
||||
"""SSE stream for real-time index update notifications.
|
||||
|
||||
Sends keepalive comments every 30s. Events:
|
||||
- ``index_updated``: partial index change (file create/modify/delete/move)
|
||||
- ``index_reloaded``: full re-index completed
|
||||
- ``vault_added``: new vault added dynamically
|
||||
- ``vault_removed``: vault removed dynamically
|
||||
"""
|
||||
queue = await sse_manager.connect()
|
||||
|
||||
async def event_generator():
|
||||
try:
|
||||
# Send initial connection event
|
||||
yield f"event: connected\ndata: {_json.dumps({'sse_clients': sse_manager.client_count})}\n\n"
|
||||
while True:
|
||||
try:
|
||||
msg = await asyncio.wait_for(queue.get(), timeout=30.0)
|
||||
yield f"event: {msg['event']}\ndata: {msg['data']}\n\n"
|
||||
except TimeoutError:
|
||||
# Keepalive comment
|
||||
yield ": keepalive\n\n"
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
finally:
|
||||
sse_manager.disconnect(queue)
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/ws/collab/{vault_name}/{path:path}")
|
||||
async def collab_websocket(websocket: WebSocket, vault_name: str, path: str):
|
||||
"""Real-time collaborative editing over WebSocket (ROADMAP #62).
|
||||
|
||||
One *room* is created per ``vault::path``; all clients editing the same
|
||||
file share Yjs/CRDT updates, awareness (cursors/selection) and a debounced
|
||||
server-side persistence of the markdown content.
|
||||
|
||||
Authentication is performed manually (FastAPI ``Depends`` do not run for
|
||||
WebSocket routes) and vault access is enforced per connection.
|
||||
"""
|
||||
from backend.services.errors import ServiceError
|
||||
|
||||
user = authenticate_websocket(websocket)
|
||||
if user is None:
|
||||
await websocket.close(code=4401)
|
||||
return
|
||||
|
||||
if not check_vault_access(vault_name, user):
|
||||
await websocket.close(code=4403)
|
||||
return
|
||||
|
||||
try:
|
||||
vault_root = get_vault_root(vault_name)
|
||||
file_path = resolve_safe_path(vault_root, path)
|
||||
except ServiceError:
|
||||
await websocket.close(code=4404)
|
||||
return
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
await websocket.close(code=4404)
|
||||
return
|
||||
|
||||
await websocket.accept()
|
||||
await collab_manager.connect(websocket, vault_name, path, file_path, user)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Scheduled tasks endpoints (#170).
|
||||
|
||||
Tasks reuse the existing mutation/notification services — this router only
|
||||
validates, persists and triggers. File-writing actions check vault access
|
||||
at creation time; the background tick re-checks nothing (system context) but
|
||||
records failures and notifies on ``schedule_failure`` (#168).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend import scheduler as _scheduler
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.schemas import StatusResponse
|
||||
|
||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||
|
||||
|
||||
class ScheduledTask(BaseModel):
|
||||
"""A programmed automatic task."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
id: str = Field(description="Task id")
|
||||
name: str = Field(description="Display name")
|
||||
action: dict[str, Any] = Field(description="{kind, params}")
|
||||
schedule: dict[str, Any] = Field(description="{kind, ...}")
|
||||
enabled: bool = Field(description="Whether the tick executes it")
|
||||
created_by: str = Field(description="Owner username")
|
||||
created_at: str = Field(description="ISO-8601 creation time")
|
||||
last_run_at: str | None = Field(default=None)
|
||||
last_status: str | None = Field(default=None)
|
||||
last_error: str | None = Field(default=None)
|
||||
run_count: int = Field(default=0)
|
||||
next_run_at: str = Field(description="ISO-8601 next due time")
|
||||
|
||||
|
||||
class TaskRunResult(BaseModel):
|
||||
"""Outcome of a manual or due run."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
task_id: str = Field(description="Task id")
|
||||
ok: bool = Field(description="True on success")
|
||||
result: dict[str, Any] | None = Field(default=None)
|
||||
error: str | None = Field(default=None)
|
||||
|
||||
|
||||
def _check_action_vault(action: dict[str, Any], user: dict[str, Any]) -> None:
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
kind = (action or {}).get("kind")
|
||||
params = (action or {}).get("params") or {}
|
||||
if kind in ("create_file", "append_to_file"):
|
||||
vault = str(params.get("vault") or "")
|
||||
if not check_vault_access(vault, user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
try:
|
||||
get_vault_root(vault)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(404, f"Unknown vault '{vault}'") from e
|
||||
|
||||
|
||||
@router.get("/tasks", response_model=list[ScheduledTask])
|
||||
async def api_scheduler_list(current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""List scheduled tasks (newest first)."""
|
||||
return _scheduler.list_tasks()
|
||||
|
||||
|
||||
@router.post("/tasks", response_model=ScheduledTask)
|
||||
async def api_scheduler_create(body: dict = Body(...), current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Create a task (``{name, action, schedule, enabled?}``)."""
|
||||
action = dict(body.get("action") or {})
|
||||
_check_action_vault(action, current_user)
|
||||
try:
|
||||
return _scheduler.create_task(
|
||||
str(body.get("name") or ""),
|
||||
action,
|
||||
dict(body.get("schedule") or {}),
|
||||
created_by=str(current_user.get("username", "api")),
|
||||
enabled=bool(body.get("enabled", True)),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/tasks/{task_id}", response_model=ScheduledTask)
|
||||
async def api_scheduler_update(task_id: str, body: dict = Body(...), current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Update a task (name / enabled / action / schedule)."""
|
||||
if "action" in body:
|
||||
_check_action_vault(dict(body["action"] or {}), current_user)
|
||||
try:
|
||||
result = _scheduler.update_task(task_id, body)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if result is None:
|
||||
raise HTTPException(404, "Task not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/tasks/{task_id}", response_model=StatusResponse)
|
||||
async def api_scheduler_delete(task_id: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Delete a task."""
|
||||
if not _scheduler.delete_task(task_id):
|
||||
raise HTTPException(404, "Task not found")
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/run", response_model=TaskRunResult)
|
||||
async def api_scheduler_run(task_id: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Execute a task immediately (manual run)."""
|
||||
try:
|
||||
return _scheduler.run_task(task_id, manual=True)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Task not found") from None
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Search, suggest, graph & index-reload endpoints (ROADMAP #85, tranche 5).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins, mêmes modèles de réponse (déménagés dans
|
||||
:mod:`backend.schemas`), mêmes dépendances d'authentification. La logique
|
||||
métier vit déjà dans :mod:`backend.services.search`,
|
||||
:mod:`backend.search`, :mod:`backend.services.graph` et
|
||||
:mod:`backend.services.mutations`.
|
||||
|
||||
Adaptations strictement équivalentes :
|
||||
- Le pool ``_search_executor`` de ``main`` vit désormais dans
|
||||
:mod:`backend.search_executor` (même dimensionnement, même cycle de vie
|
||||
géré par le lifespan de ``main``) : accès via
|
||||
:func:`get_search_executor`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from backend.audit import log_file_save
|
||||
from backend.auth.middleware import check_vault_access, require_admin, require_auth
|
||||
from backend.indexer import get_vault_data, reload_index, update_single_file
|
||||
from backend.schemas import (
|
||||
AdvancedSearchResponse,
|
||||
GraphResponse,
|
||||
ReloadResponse,
|
||||
ReplaceResponse,
|
||||
SearchResponse,
|
||||
SuggestResponse,
|
||||
TagsResponse,
|
||||
TagSuggestResponse,
|
||||
TreeSearchResponse,
|
||||
VaultPathsResponse,
|
||||
VaultStatsResponse,
|
||||
)
|
||||
from backend.search import suggest_tags, suggest_titles
|
||||
from backend.search_executor import get_search_executor
|
||||
from backend.services.graph import get_graph as service_get_graph
|
||||
from backend.services.mutations import (
|
||||
replace_in_files as service_replace_in_files,
|
||||
)
|
||||
from backend.services.search import advanced_search_vaults, list_paths, search_paths, search_vaults
|
||||
from backend.services.search import list_tags as service_list_tags
|
||||
from backend.sse import sse_manager
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
router = APIRouter(tags=["search"])
|
||||
|
||||
|
||||
@router.get("/api/search", response_model=SearchResponse)
|
||||
async def api_search(
|
||||
q: str = Query("", description="Search query"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
tag: str | None = Query(None, description="Tag filter"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Results per page"),
|
||||
offset: int = Query(0, ge=0, description="Pagination offset"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Full-text search across vaults with relevance scoring.
|
||||
|
||||
Supports combining free-text queries with tag filters.
|
||||
Results are ranked by a multi-factor scoring algorithm.
|
||||
Pagination via ``limit`` and ``offset`` (defaults preserve backward compat).
|
||||
|
||||
Args:
|
||||
q: Free-text search string.
|
||||
vault: Vault name or ``"all"`` to search everywhere.
|
||||
tag: Comma-separated tag names to require.
|
||||
limit: Max results per page (1–200).
|
||||
offset: Pagination offset.
|
||||
|
||||
Returns:
|
||||
``SearchResponse`` with ranked results and snippets.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
# Fetch the full result set (capped at DEFAULT_SEARCH_LIMIT internally) and
|
||||
# paginate in the shared service so routes and tools share the same logic.
|
||||
return await loop.run_in_executor(
|
||||
get_search_executor(),
|
||||
# #194 : filtre par vault accessible AVANT pagination ("*" sans les
|
||||
# dossiers persos) — sinon un user voyait les notes des autres.
|
||||
# #196 : les documents reçus par partage dirigé sont ajoutés aux
|
||||
# résultats (vault virtuel "home-<user>/Partage").
|
||||
partial(
|
||||
search_vaults, q, vault, tag, limit, offset,
|
||||
is_allowed=lambda v: check_vault_access(v, current_user),
|
||||
username=current_user["username"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/tags", response_model=TagsResponse)
|
||||
async def api_tags(vault: str | None = Query(None, description="Vault filter"), current_user=Depends(require_auth)):
|
||||
"""Return all unique tags with occurrence counts.
|
||||
|
||||
Args:
|
||||
vault: Optional vault name to restrict tag aggregation.
|
||||
|
||||
Returns:
|
||||
``TagsResponse`` with tags sorted by descending count.
|
||||
"""
|
||||
return {"vault_filter": vault, "tags": service_list_tags(vault)}
|
||||
|
||||
|
||||
@router.get("/api/tree-search", response_model=TreeSearchResponse)
|
||||
async def api_tree_search(
|
||||
q: str = Query("", description="Search query"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Search for files and directories in the tree structure using pre-built index.
|
||||
|
||||
Uses the in-memory path index for instant filtering without filesystem access.
|
||||
|
||||
Args:
|
||||
q: Search string to match against file/directory paths.
|
||||
vault: Vault name or "all" to search everywhere.
|
||||
|
||||
Returns:
|
||||
``TreeSearchResponse`` with matching paths.
|
||||
"""
|
||||
return search_paths(q, vault)
|
||||
|
||||
|
||||
@router.get("/api/vault/{vault_name}/paths", response_model=VaultPathsResponse)
|
||||
async def api_vault_paths(
|
||||
vault_name: str,
|
||||
limit: int = Query(5000, ge=1, le=20000, description="Maximum number of indexed paths to return"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return a flat list of every indexed file and directory in a vault.
|
||||
|
||||
Used by the AI assistant ``@`` mention menu to filter paths instantly on
|
||||
the client (one request instead of one per keystroke).
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
limit: Maximum number of entries returned.
|
||||
|
||||
Returns:
|
||||
``VaultPathsResponse`` with the vault's indexed paths.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
return list_paths(vault_name, limit=limit)
|
||||
|
||||
|
||||
@router.get("/api/search/advanced", response_model=AdvancedSearchResponse)
|
||||
async def api_advanced_search(
|
||||
q: str = Query("", description="Advanced search query (supports tag:, vault:, title:, path:, ext: operators)"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
tag: str | None = Query(None, description="Comma-separated tag filter"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Results per page"),
|
||||
offset: int = Query(0, ge=0, description="Pagination offset"),
|
||||
sort: str = Query("relevance", description="Sort by 'relevance' or 'modified'"),
|
||||
case_sensitive: bool = Query(False, description="Match case"),
|
||||
whole_word: bool = Query(False, description="Match whole words only"),
|
||||
regex: bool = Query(False, description="Treat query as regex"),
|
||||
include_paths: str | None = Query(None, description="Comma-separated glob patterns to include"),
|
||||
exclude_paths: str | None = Query(None, description="Comma-separated glob patterns to exclude"),
|
||||
created: str | None = Query(None, description="Created date filter (>date, <date, date..date)"),
|
||||
modified: str | None = Query(None, description="Modified date filter (>date, <date, date..date, <Nd)"),
|
||||
size: str | None = Query(None, description="Size filter (>size, <size, size..size, e.g. >1MB, <10KB)"),
|
||||
semantic: bool = Query(False, description="Fuse TF-IDF with semantic embeddings (RRF)"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Advanced full-text search with TF-IDF scoring, facets, and pagination.
|
||||
|
||||
Supports advanced query operators:
|
||||
- ``tag:<name>`` or ``#<name>`` — filter by tag
|
||||
- ``vault:<name>`` — filter by vault
|
||||
- ``title:<text>`` — filter by title substring
|
||||
- ``path:<text>`` — filter by path substring
|
||||
- ``ext:<type>`` — filter by file extension
|
||||
- ``created:>2024-01-01`` — filter by creation date
|
||||
- ``modified:<7d`` or ``modified:2024-01-01..2024-06-01`` — filter by modification date
|
||||
- ``size:>1MB`` or ``size:100KB..1MB`` — filter by file size
|
||||
- Remaining text is scored using TF-IDF with accent normalization.
|
||||
- Toggles: case_sensitive, whole_word, regex
|
||||
- Path filters: include_paths, exclude_paths (glob patterns)
|
||||
- ``semantic=true`` — fuse the TF-IDF ranking with the semantic (embedding)
|
||||
ranking via Reciprocal Rank Fusion and expose ``semantic_score`` per result.
|
||||
|
||||
Results include ``<mark>``-highlighted snippets and faceted tag/vault counts.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
search_fn = partial(advanced_search_vaults, q, vault=vault, tag=tag,
|
||||
limit=limit, offset=offset, sort=sort,
|
||||
case_sensitive=case_sensitive, whole_word=whole_word, regex=regex,
|
||||
include_paths=include_paths, exclude_paths=exclude_paths,
|
||||
created=created, modified=modified, size=size, semantic=semantic)
|
||||
try:
|
||||
return await loop.run_in_executor(get_search_executor(), search_fn)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/search/replace", response_model=ReplaceResponse)
|
||||
async def api_search_replace(
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Find and replace across vault files."""
|
||||
query = body.get("query", "")
|
||||
replacement = body.get("replacement", "")
|
||||
vault_filter = body.get("vault", "all")
|
||||
case_sensitive = body.get("case_sensitive", False)
|
||||
whole_word = body.get("whole_word", False)
|
||||
regex_mode = body.get("regex", False)
|
||||
include_paths = body.get("include_paths")
|
||||
exclude_paths = body.get("exclude_paths")
|
||||
replace_all = body.get("replace_all", False)
|
||||
dry_run = body.get("dry_run", not replace_all)
|
||||
|
||||
if not query:
|
||||
raise HTTPException(400, "Query is required")
|
||||
|
||||
result = service_replace_in_files(
|
||||
query,
|
||||
replacement,
|
||||
vault=vault_filter,
|
||||
case_sensitive=case_sensitive,
|
||||
whole_word=whole_word,
|
||||
regex=regex_mode,
|
||||
include_paths=include_paths,
|
||||
exclude_paths=exclude_paths,
|
||||
replace_all=replace_all,
|
||||
dry_run=dry_run,
|
||||
is_vault_allowed=lambda v: check_vault_access(v, current_user),
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return result
|
||||
|
||||
# Side effects for applied replacements (audit + incremental index).
|
||||
for match in result.get("replaced", []):
|
||||
log_file_save(current_user["username"], match["vault"], match["path"], match.get("size", 0))
|
||||
vault_data = get_vault_data(match["vault"])
|
||||
if vault_data:
|
||||
abs_path = str(Path(vault_data["path"]) / match["path"])
|
||||
await update_single_file(match["vault"], abs_path)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/api/suggest", response_model=SuggestResponse)
|
||||
async def api_suggest(
|
||||
q: str = Query("", description="Prefix to search for in file titles"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
limit: int = Query(10, ge=1, le=50, description="Max suggestions"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Suggest file titles matching a prefix (accent-insensitive).
|
||||
|
||||
Used for autocomplete in the search input.
|
||||
|
||||
Args:
|
||||
q: User-typed prefix (minimum 2 characters).
|
||||
vault: Vault name or ``"all"``.
|
||||
limit: Max number of suggestions.
|
||||
|
||||
Returns:
|
||||
``SuggestResponse`` with matching file title suggestions.
|
||||
"""
|
||||
suggestions = suggest_titles(q, vault_filter=vault, limit=limit)
|
||||
return {"query": q, "suggestions": suggestions}
|
||||
|
||||
|
||||
@router.get("/api/tags/suggest", response_model=TagSuggestResponse)
|
||||
async def api_tags_suggest(
|
||||
q: str = Query("", description="Prefix to search for in tags"),
|
||||
vault: str = Query("all", description="Vault filter"),
|
||||
limit: int = Query(10, ge=1, le=50, description="Max suggestions"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Suggest tags matching a prefix (accent-insensitive).
|
||||
|
||||
Used for autocomplete when typing ``tag:`` or ``#`` in the search input.
|
||||
|
||||
Args:
|
||||
q: User-typed prefix (with or without ``#``, minimum 2 characters).
|
||||
vault: Vault name or ``"all"``.
|
||||
limit: Max number of suggestions.
|
||||
|
||||
Returns:
|
||||
``TagSuggestResponse`` with matching tag suggestions and counts.
|
||||
"""
|
||||
suggestions = suggest_tags(q, vault_filter=vault, limit=limit)
|
||||
return {"query": q, "suggestions": suggestions}
|
||||
|
||||
|
||||
@router.get("/api/index/reload", response_model=ReloadResponse)
|
||||
async def api_reload(current_user=Depends(require_admin)):
|
||||
"""Force a full re-index of all configured vaults.
|
||||
|
||||
Returns:
|
||||
``ReloadResponse`` with per-vault file and tag counts.
|
||||
"""
|
||||
stats = await reload_index()
|
||||
await sse_manager.broadcast("index_reloaded", {
|
||||
"vaults": list(stats.keys()),
|
||||
"stats": stats,
|
||||
})
|
||||
return {"status": "ok", "vaults": stats}
|
||||
|
||||
|
||||
@router.get("/api/graph/{vault_name}", response_model=GraphResponse)
|
||||
async def api_graph(
|
||||
vault_name: str,
|
||||
path: str = Query("", description="Relative path to focus on"),
|
||||
depth: int = Query(1, ge=0, le=3, description="How many levels deep to expand"),
|
||||
scope: str = Query("directory", description="'directory' (default) or 'full' for entire vault"),
|
||||
tag: str = Query("", description="Filter: only show files with this tag"),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Return graph data (nodes and edges) for a vault or directory.
|
||||
|
||||
Nodes represent files and directories. Edges represent parent-child
|
||||
relationships and wikilinks between markdown files.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative directory path to focus on (empty = root).
|
||||
depth: Expansion depth (0 = only direct children, 1-3 = deeper).
|
||||
scope: 'directory' for subtree, 'full' for entire vault.
|
||||
tag: Optional tag filter (only files with this tag appear).
|
||||
|
||||
Returns:
|
||||
``GraphResponse`` with nodes and edges.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'")
|
||||
|
||||
return service_get_graph(vault_name, path=path, depth=depth, scope=scope, tag=tag)
|
||||
|
||||
|
||||
@router.get("/api/index/reload/{vault_name}", response_model=VaultStatsResponse)
|
||||
async def api_reload_vault(vault_name: str, current_user=Depends(require_admin)):
|
||||
"""Force a re-index of a single vault.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault to reindex.
|
||||
|
||||
Returns:
|
||||
Dict with vault statistics.
|
||||
"""
|
||||
try:
|
||||
from backend.indexer import reload_single_vault
|
||||
stats = await reload_single_vault(vault_name)
|
||||
await sse_manager.broadcast("vault_reloaded", {
|
||||
"vault": vault_name,
|
||||
"stats": stats,
|
||||
})
|
||||
return {"status": "ok", "vault": vault_name, "stats": stats}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Public share endpoints (ROADMAP #85, tranche 3).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/share/*``, ``/api/shares``,
|
||||
``/s/{token}*``), mêmes modèles de réponse, mêmes dépendances
|
||||
d'authentification (les pages ``/s/*`` restent publiques). La logique
|
||||
métier vit déjà dans :mod:`backend.share`.
|
||||
|
||||
Adaptations strictement équivalentes (pas de changement de comportement) :
|
||||
- ``_resolve_safe_path`` / ``_backup_file`` de ``main`` n'étaient que des
|
||||
wrappers directs : appelés ici via :mod:`backend.services.paths` et
|
||||
:mod:`backend.services.backups` (mêmes signatures, mêmes exceptions
|
||||
``ServiceError`` toujours mappées par le handler global de ``main``).
|
||||
- ``_render_markdown`` vient de :mod:`backend.render` (#85 T9, sans cycle
|
||||
d'import).
|
||||
"""
|
||||
|
||||
import html as html_mod
|
||||
import json as _json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import frontmatter
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, Response
|
||||
|
||||
from backend.auth.middleware import check_vault_access, get_current_user, require_auth
|
||||
from backend.auth.user_store import get_user
|
||||
from backend.indexer import get_vault_data, parse_markdown_file, update_single_file
|
||||
from backend.render import _render_markdown
|
||||
from backend.schemas import ShareModel, StatusResponse
|
||||
from backend.secret_redactor import redact_file_content
|
||||
from backend.services.backups import create_backup
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.share import (
|
||||
create_share,
|
||||
get_share_by_token,
|
||||
list_shares,
|
||||
record_access,
|
||||
revoke_share,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate")
|
||||
|
||||
# Lazy import: WeasyPrint PDF export (requires GTK, may not be available everywhere)
|
||||
try:
|
||||
from backend.pdf_export import build_pdf_html, generate_pdf
|
||||
except Exception: # pragma: no cover - WeasyPrint/GTK missing
|
||||
generate_pdf = None # type: ignore[assignment]
|
||||
build_pdf_html = None # type: ignore[assignment]
|
||||
|
||||
logging.getLogger("obsigate").warning("PDF export unavailable (WeasyPrint/GTK not found)")
|
||||
|
||||
router = APIRouter(tags=["sharing"])
|
||||
|
||||
|
||||
def _gate_share(share: dict | None, user: dict | None) -> dict:
|
||||
"""#196 — directed shares: auth + membership check for ``/s/*`` pages.
|
||||
|
||||
Public share (``shared_with`` empty) → pass-through, no auth required.
|
||||
Directed share → requires an authenticated user who is the creator, an
|
||||
admin, or listed in ``shared_with``. Anything else answers **404** (never
|
||||
403) so an outsider cannot learn that a token exists.
|
||||
"""
|
||||
if share is None:
|
||||
raise HTTPException(404, "Share not found or expired")
|
||||
recipients = share.get("shared_with") or []
|
||||
if not recipients:
|
||||
return share
|
||||
if user is None:
|
||||
raise HTTPException(404, "Share not found or expired")
|
||||
if user["username"] == share.get("created_by") or user.get("role") == "admin" \
|
||||
or user["username"] in recipients:
|
||||
return share
|
||||
raise HTTPException(404, "Share not found or expired")
|
||||
|
||||
|
||||
@router.post("/api/share/{vault_name}", response_model=ShareModel)
|
||||
async def api_share_create(
|
||||
vault_name: str,
|
||||
body: dict = Body(...),
|
||||
current_user=Depends(require_auth),
|
||||
):
|
||||
"""Create a public share link for a document.
|
||||
|
||||
Also sets ``publish: true`` in the file's YAML frontmatter so the
|
||||
frontend can visually indicate the file is publicly shared.
|
||||
"""
|
||||
if not check_vault_access(vault_name, current_user):
|
||||
raise HTTPException(403, f"Accès refusé à la vault '{vault_name}'")
|
||||
path = body.get("path") or ""
|
||||
if not path:
|
||||
raise HTTPException(400, "Chemin de fichier requis")
|
||||
expires = body.get("expires_in_hours")
|
||||
# #196 — directed share: validate recipients before creating anything.
|
||||
recipients = body.get("shared_with") or []
|
||||
if not isinstance(recipients, list):
|
||||
raise HTTPException(400, "shared_with doit être une liste d'utilisateurs")
|
||||
for name in recipients:
|
||||
if not isinstance(name, str) or not get_user(name):
|
||||
raise HTTPException(400, f"Utilisateur inconnu : {name}")
|
||||
share = create_share(vault_name, path, current_user["username"], expires, recipients)
|
||||
share["url"] = f"/s/{share['token']}"
|
||||
|
||||
# Set publish: true in the file's frontmatter
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if vault_data:
|
||||
file_path = resolve_safe_path(Path(vault_data["path"]), path)
|
||||
if file_path.exists() and file_path.suffix == ".md":
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
post = frontmatter.loads(raw)
|
||||
if not post.metadata.get("publish"):
|
||||
post.metadata["publish"] = True
|
||||
new_raw = frontmatter.dumps(post)
|
||||
create_backup(file_path, vault_name, path)
|
||||
file_path.write_text(new_raw, encoding="utf-8")
|
||||
await update_single_file(vault_name, str(file_path))
|
||||
logger.info(f"Set publish:true on {vault_name}/{path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to set publish metadata on {vault_name}/{path}: {e}")
|
||||
|
||||
return share
|
||||
|
||||
|
||||
@router.get("/api/shares", response_model=list[ShareModel])
|
||||
async def api_shares_list(vault: str | None = Query(None), current_user=Depends(require_auth)):
|
||||
"""List shares (optionally filtered by vault).
|
||||
|
||||
#196 : un non-admin ne voit que SES partages (créés ou reçus) ; un admin
|
||||
voit tout (comportement historique).
|
||||
"""
|
||||
username = None if current_user.get("role") == "admin" else current_user["username"]
|
||||
shares = list_shares(vault, username)
|
||||
for s in shares:
|
||||
s["url"] = f"/s/{s['token']}"
|
||||
return shares
|
||||
|
||||
|
||||
@router.delete("/api/share/{share_id}", response_model=StatusResponse)
|
||||
async def api_share_revoke(share_id: str, current_user=Depends(require_auth)):
|
||||
# #196 — only the creator or an admin may revoke; a recipient cannot.
|
||||
from backend.share import _read
|
||||
share = _read()["shares"].get(share_id)
|
||||
if share and current_user.get("role") != "admin" \
|
||||
and share.get("created_by") != current_user["username"]:
|
||||
raise HTTPException(403, "Seul le créateur du partage peut le révoquer")
|
||||
if not revoke_share(share_id):
|
||||
raise HTTPException(404, "Share not found")
|
||||
return {"status": "revoked"}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/s/{token}/pdf",
|
||||
response_class=Response,
|
||||
responses={200: {"content": {"application/pdf": {}}, "description": "Shared document as PDF"}},
|
||||
)
|
||||
async def public_share_pdf_download(token: str, current_user=Depends(get_current_user)):
|
||||
"""Download shared document as real PDF via WeasyPrint."""
|
||||
if generate_pdf is None:
|
||||
raise HTTPException(501, "PDF export unavailable (WeasyPrint/GTK not available)")
|
||||
share = get_share_by_token(token)
|
||||
share = _gate_share(share, current_user)
|
||||
vault_data = get_vault_data(share["vault"])
|
||||
if not vault_data:
|
||||
raise HTTPException(404, "Vault not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, share["path"])
|
||||
if not file_path.exists():
|
||||
raise HTTPException(404, "File not found")
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
raise HTTPException(500, "Cannot read file")
|
||||
record_access(token)
|
||||
raw = redact_file_content(raw, str(file_path))
|
||||
post = parse_markdown_file(raw)
|
||||
ext = file_path.suffix.lower()
|
||||
if ext == ".md":
|
||||
html = _render_markdown(post.content, share["vault"], file_path)
|
||||
else:
|
||||
html = f'<pre style="font-family:monospace;font-size:12px;line-height:1.6;white-space:pre-wrap">{html_mod.escape(raw)}</pre>'
|
||||
title = post.metadata.get("title", file_path.stem)
|
||||
pdf_html = build_pdf_html(html, str(title))
|
||||
pdf_bytes = generate_pdf(pdf_html, str(title))
|
||||
safe_name = "".join(c for c in str(title) if c.isascii() and (c.isalnum() or c in " _-.")).strip() or "document"
|
||||
return Response(content=pdf_bytes, media_type="application/pdf", headers={"Content-Disposition": f'attachment; filename="{safe_name}.pdf"'})
|
||||
|
||||
|
||||
@router.get("/s/{token}/raw", response_class=FileResponse)
|
||||
async def public_share_raw(token: str, current_user=Depends(get_current_user)):
|
||||
"""Download the raw (original) shared document."""
|
||||
share = get_share_by_token(token)
|
||||
share = _gate_share(share, current_user)
|
||||
vault_data = get_vault_data(share["vault"])
|
||||
if not vault_data:
|
||||
raise HTTPException(404, "Vault not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, share["path"])
|
||||
if not file_path.exists():
|
||||
raise HTTPException(404, "File not found")
|
||||
record_access(token)
|
||||
return FileResponse(path=str(file_path), filename=file_path.name, media_type="application/octet-stream")
|
||||
|
||||
|
||||
@router.get("/s/{token}", response_class=HTMLResponse)
|
||||
async def public_share_view(request: Request, token: str, current_user=Depends(get_current_user)):
|
||||
"""Public share view — no authentication required (unless directed #196)."""
|
||||
from backend.csp import inject_csp_nonce
|
||||
|
||||
share = get_share_by_token(token)
|
||||
share = _gate_share(share, current_user)
|
||||
vault_data = get_vault_data(share["vault"])
|
||||
if not vault_data:
|
||||
raise HTTPException(404, "Vault not found")
|
||||
vault_root = Path(vault_data["path"])
|
||||
file_path = resolve_safe_path(vault_root, share["path"])
|
||||
if not file_path.exists():
|
||||
raise HTTPException(404, "File not found")
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
raise HTTPException(500, "Cannot read file")
|
||||
record_access(token)
|
||||
raw = redact_file_content(raw, str(file_path))
|
||||
post = parse_markdown_file(raw)
|
||||
ext = file_path.suffix.lower()
|
||||
|
||||
if ext == ".md":
|
||||
html = _render_markdown(post.content, share["vault"], file_path)
|
||||
else:
|
||||
escaped = html_mod.escape(raw)
|
||||
html = f'<pre style="background:var(--bg-card);border:1px solid var(--border);border-radius:8px;padding:16px;overflow-x:auto;font-size:0.85rem;line-height:1.6"><code>{escaped}</code></pre>'
|
||||
|
||||
title = post.metadata.get("title", file_path.stem)
|
||||
|
||||
# Escape everything user-controlled before embedding in HTML/JS (BUG-022).
|
||||
title_esc = html_mod.escape(str(title))
|
||||
# Neutralise ``</script>`` in the JS string literal too.
|
||||
title_download_js = (
|
||||
_json.dumps(f"{title}.md")
|
||||
.replace("<", "\\u003c")
|
||||
.replace(">", "\\u003e")
|
||||
.replace("&", "\\u0026")
|
||||
)
|
||||
|
||||
# JSON-escape raw content for embedding in HTML, and neutralise ``</script>``.
|
||||
raw_json = (
|
||||
_json.dumps(raw)
|
||||
.replace("<", "\\u003c")
|
||||
.replace(">", "\\u003e")
|
||||
.replace("&", "\\u0026")
|
||||
)
|
||||
fm_html = ""
|
||||
if post.metadata:
|
||||
fm_items = []
|
||||
skip_keys = {"title", "titre"}
|
||||
for k, v in post.metadata.items():
|
||||
if k in skip_keys:
|
||||
continue
|
||||
if isinstance(v, list):
|
||||
v = ", ".join(str(x) for x in v)
|
||||
elif isinstance(v, bool):
|
||||
v = "✓" if v else "✗"
|
||||
elif v is None:
|
||||
v = "—"
|
||||
fm_items.append(
|
||||
f'<div class="fm-row"><span class="fm-key">{html_mod.escape(str(k))}</span>'
|
||||
f'<span class="fm-val">{html_mod.escape(str(v))}</span></div>'
|
||||
)
|
||||
if fm_items:
|
||||
fm_html = f'<div class="fm-section"><div class="fm-header">Frontmatter</div><div class="fm-body">{"".join(fm_items)}</div></div>'
|
||||
|
||||
return HTMLResponse(
|
||||
inject_csp_nonce(
|
||||
f"""<!DOCTYPE html><html lang="fr" data-theme="dark"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>{title_esc} — ObsiGate Share</title>
|
||||
<style>
|
||||
:root {{ --bg:#1a1a2e; --bg-card:#16213e; --text:#e0e0e0; --text-muted:#888; --accent:#6366f1; --border:#2a2a4a; --banner-bg:var(--accent); --banner-text:#fff; }}
|
||||
[data-theme="light"] {{ --bg:#f8f9fa; --bg-card:#fff; --text:#1a1a2e; --text-muted:#666; --accent:#4f46e5; --border:#ddd; --banner-bg:#eef2ff; --banner-text:#4338ca; }}
|
||||
*{{box-sizing:border-box;margin:0;padding:0}}
|
||||
body{{font-family:system-ui,-apple-system,sans-serif;background:var(--bg);color:var(--text);line-height:1.7;min-height:100vh}}
|
||||
.toolbar{{position:sticky;top:0;z-index:10;background:var(--bg-card);border-bottom:1px solid var(--border);padding:8px 16px;display:flex;align-items:center;gap:8px;flex-wrap:wrap}}
|
||||
.toolbar-title{{font-weight:600;font-size:0.9rem;margin-right:auto;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}}
|
||||
.toolbar-btn{{padding:6px 12px;border:1px solid var(--border);border-radius:6px;background:var(--bg);color:var(--text);cursor:pointer;font-size:0.8rem;display:flex;align-items:center;gap:5px;transition:all .15s}}
|
||||
.toolbar-btn:hover{{background:var(--accent);color:#fff;border-color:var(--accent)}}
|
||||
.toolbar-btn svg{{width:15px;height:15px;flex-shrink:0}}
|
||||
.toolbar-btn:hover svg{{stroke:#fff}}
|
||||
.share-banner{{background:var(--banner-bg);color:var(--banner-text);padding:6px 16px;font-size:0.8rem;text-align:center;display:flex;align-items:center;justify-content:center;gap:6px}}
|
||||
.share-banner svg{{width:14px;height:14px;flex-shrink:0}}
|
||||
.content{{max-width:820px;margin:0 auto;padding:24px 20px 60px}}
|
||||
.content h1{{font-size:1.8rem;margin-bottom:16px;border-bottom:2px solid var(--border);padding-bottom:8px}}
|
||||
.content h2{{font-size:1.4rem;margin:24px 0 12px}}
|
||||
.content h3{{font-size:1.15rem;margin:20px 0 8px}}
|
||||
.content p{{margin:8px 0}}
|
||||
.content pre{{background:var(--bg-card);border:1px solid var(--border);border-radius:8px;padding:12px 16px;overflow-x:auto;font-size:0.85rem}}
|
||||
.content code{{font-size:0.9em;background:var(--bg-card);padding:1px 4px;border-radius:3px}}
|
||||
.content pre code{{background:none;padding:0}}
|
||||
.content a{{color:var(--accent)}}.content img{{max-width:100%;border-radius:6px}}
|
||||
.fm-section{{background:var(--bg-card);border:1px solid var(--border);border-radius:8px;padding:12px 16px;margin-bottom:20px}}
|
||||
.fm-header{{font-weight:600;font-size:0.8rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.5px;margin-bottom:8px}}
|
||||
.fm-body{{display:grid;grid-template-columns:1fr 2fr;gap:4px 12px;font-size:0.85rem}}
|
||||
.fm-row{{display:contents}}
|
||||
.fm-key{{color:var(--accent);font-weight:500}}
|
||||
.fm-val{{color:var(--text);word-break:break-word}}
|
||||
.content blockquote{{border-left:3px solid var(--accent);padding-left:16px;color:var(--text-muted);margin:12px 0}}
|
||||
.content table{{border-collapse:collapse;width:100%;margin:12px 0}}
|
||||
.content th,.content td{{border:1px solid var(--border);padding:8px 12px;text-align:left}}
|
||||
.content th{{background:var(--bg-card)}}
|
||||
@media print{{.toolbar,.share-banner{{display:none}}body{{background:#fff;color:#000}}}}
|
||||
@media(max-width:600px){{.content{{padding:16px 12px 40px}}.toolbar{{gap:4px}}.toolbar-btn{{padding:4px 8px;font-size:0.7rem}}}}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="share-banner">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14.5 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V7.5L14.5 2z"/><polyline points="14 2 14 8 20 8"/></svg>
|
||||
Document partagé via ObsiGate
|
||||
</div>
|
||||
<div class="toolbar">
|
||||
<span class="toolbar-title">{title_esc}</span>
|
||||
<button class="toolbar-btn" data-share-theme title="Thème clair/sombre">
|
||||
<svg id="theme-icon-dark" xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"/></svg>
|
||||
<svg id="theme-icon-light" xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="display:none"><circle cx="12" cy="12" r="5"/><line x1="12" y1="1" x2="12" y2="3"/><line x1="12" y1="21" x2="12" y2="23"/><line x1="4.22" y1="4.22" x2="5.64" y2="5.64"/><line x1="18.36" y1="18.36" x2="19.78" y2="19.78"/><line x1="1" y1="12" x2="3" y2="12"/><line x1="21" y1="12" x2="23" y2="12"/><line x1="4.22" y1="19.78" x2="5.64" y2="18.36"/><line x1="18.36" y1="5.64" x2="19.78" y2="4.22"/></svg>
|
||||
</button>
|
||||
<button class="toolbar-btn" data-share-md title="Télécharger en Markdown">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" y1="15" x2="12" y2="3"/></svg>
|
||||
.md
|
||||
</button>
|
||||
<button class="toolbar-btn" data-share-pdf title="Télécharger en PDF">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2-2V8z"/><polyline points="14 2 14 8 20 8"/><line x1="16" y1="13" x2="8" y2="13"/><line x1="16" y1="17" x2="8" y2="17"/><polyline points="10 9 9 9 8 9"/></svg>
|
||||
PDF
|
||||
</button>
|
||||
</div>
|
||||
<div class="content" id="content">{fm_html}{html}</div>
|
||||
<script id="raw-content" type="text/plain" style="display:none">{raw_json}</script>
|
||||
<script>
|
||||
function toggleTheme(){{var t=document.documentElement;var isDark=t.dataset.theme==="dark";t.dataset.theme=isDark?"light":"dark";document.getElementById("theme-icon-dark").style.display=isDark?"none":"";document.getElementById("theme-icon-light").style.display=isDark?"":"none";localStorage.setItem("obsigate-share-theme",t.dataset.theme)}}
|
||||
(function(){{var s=localStorage.getItem("obsigate-share-theme");if(!s)s="dark";document.documentElement.dataset.theme=s;var isDark=s==="dark";document.getElementById("theme-icon-dark").style.display=isDark?"":"none";document.getElementById("theme-icon-light").style.display=isDark?"none":""}})();
|
||||
function exportMD(){{var raw=JSON.parse(document.getElementById("raw-content").textContent);var b=new Blob([raw],{{type:"text/markdown"}});var a=document.createElement("a");a.href=URL.createObjectURL(b);a.download={title_download_js};a.click()}}
|
||||
document.querySelector("[data-share-theme]").addEventListener("click",toggleTheme);
|
||||
document.querySelector("[data-share-md]").addEventListener("click",exportMD);
|
||||
document.querySelector("[data-share-pdf]").addEventListener("click",function(){{location.href=location.pathname+"/pdf"}});
|
||||
</script></body></html>""",
|
||||
request.state.csp_nonce,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
# backend/routers/sso.py
|
||||
# Fusion ACC-3.1 (#216) — Points d'entrée SSO de l'émettrice.
|
||||
#
|
||||
# GET /sso/authorize?return_to=<url> : point d'entrée des autres
|
||||
# applications — valide le return_to (liste blanche *.dracodev.net,
|
||||
# anti open-redirect) puis redirige vers la page de connexion
|
||||
# d'ObsiGate en conservant le paramètre ; après connexion, le
|
||||
# frontend repart vers l'application demanderesse (le cookie
|
||||
# draco_session posé à la connexion couvre tout le domaine parent).
|
||||
#
|
||||
# GET /api/sso/revocations : liste des jti révoqués, réservée aux
|
||||
# autres applications (clé machine DRACO_SSO_MACHINE_KEY, en-tête
|
||||
# X-Draco-Machine-Key) qui la mettent en cache 60 s (design §5.2).
|
||||
#
|
||||
# GET /sso/logout?return_to=<url> : symétrique de /sso/authorize —
|
||||
# point d'entrée navigable de la déconnexion globale (BUG-122,
|
||||
# fusion #141). Les autres applications y naviguent depuis leur menu
|
||||
# compte ; la déconnexion exécutée est celle de POST /api/auth/logout
|
||||
# (logique partagée : jetons natifs révoqués, jti draco révoqué,
|
||||
# cookie effacé sur le domaine parent), puis redirection vers le
|
||||
# return_to validé (même liste blanche ; repli « / »).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import RedirectResponse
|
||||
|
||||
from backend import draco_session
|
||||
from backend.auth.router import apply_global_logout
|
||||
|
||||
router = APIRouter(tags=["SSO"])
|
||||
|
||||
|
||||
@router.get("/sso/authorize")
|
||||
async def sso_authorize(return_to: str = ""):
|
||||
target = draco_session.validate_return_to(return_to)
|
||||
if target:
|
||||
return RedirectResponse(
|
||||
url=f"/?return_to={quote(target, safe='')}", status_code=302)
|
||||
return RedirectResponse(url="/", status_code=302)
|
||||
|
||||
|
||||
@router.get("/sso/logout")
|
||||
async def sso_logout(request: Request, return_to: str = ""):
|
||||
target = draco_session.validate_return_to(return_to)
|
||||
response = RedirectResponse(url=target or "/", status_code=302)
|
||||
apply_global_logout(request, response)
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/api/sso/revocations")
|
||||
async def sso_revocations(request: Request):
|
||||
machine_key = os.environ.get("DRACO_SSO_MACHINE_KEY", "")
|
||||
if not machine_key:
|
||||
raise HTTPException(404, "Non configuré")
|
||||
if request.headers.get("X-Draco-Machine-Key") != machine_key:
|
||||
raise HTTPException(401, "Clé machine invalide")
|
||||
return {"revoked": draco_session.revoked_list()}
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Vault management endpoints (ROADMAP #85, tranche 8).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/vaults*``), mêmes modèles de réponse
|
||||
(``VaultInfo`` déménagé dans :mod:`backend.schemas`), mêmes dépendances
|
||||
d'authentification.
|
||||
|
||||
Le handle du file-watcher vit désormais dans :mod:`backend.watcher_state`
|
||||
(partagé avec le lifespan de ``main``) au lieu du global de ``main``.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
|
||||
from backend.auth.middleware import require_admin, require_auth
|
||||
from backend.indexer import add_vault_to_index, index, remove_vault_from_index
|
||||
from backend.schemas import VaultActionResponse, VaultInfo, VaultsStatusResponse, VaultStatsResponse
|
||||
from backend.services.vaults import list_accessible_vaults
|
||||
from backend.sse import sse_manager
|
||||
from backend.watcher_state import get_watcher
|
||||
|
||||
router = APIRouter(tags=["vaults"])
|
||||
|
||||
|
||||
@router.get("/api/vaults", response_model=list[VaultInfo])
|
||||
async def api_vaults(current_user=Depends(require_auth)):
|
||||
"""List configured vaults the user has access to.
|
||||
|
||||
Returns:
|
||||
List of vault summary objects filtered by user permissions.
|
||||
"""
|
||||
return list_accessible_vaults(current_user)
|
||||
|
||||
|
||||
@router.post("/api/vaults/add", response_model=VaultStatsResponse)
|
||||
async def api_add_vault(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Add a new vault dynamically without restarting.
|
||||
|
||||
Body:
|
||||
name: Display name for the vault.
|
||||
path: Absolute filesystem path to the vault directory.
|
||||
"""
|
||||
name = body.get("name", "").strip()
|
||||
vault_path = body.get("path", "").strip()
|
||||
|
||||
if not name or not vault_path:
|
||||
raise HTTPException(status_code=400, detail="Both 'name' and 'path' are required")
|
||||
|
||||
if name in index:
|
||||
raise HTTPException(status_code=409, detail=f"Vault '{name}' already exists")
|
||||
|
||||
if not Path(vault_path).exists():
|
||||
raise HTTPException(status_code=400, detail=f"Path does not exist: {vault_path}")
|
||||
|
||||
stats = await add_vault_to_index(name, vault_path)
|
||||
|
||||
# #194 : persister, sinon ce vault disparaît au prochain rebuild/redémarrage.
|
||||
from backend.indexer import persist_vault
|
||||
|
||||
persist_vault(name)
|
||||
|
||||
# Start watching the new vault
|
||||
watcher = get_watcher()
|
||||
if watcher:
|
||||
await watcher.add_vault(name, vault_path)
|
||||
|
||||
await sse_manager.broadcast("vault_added", {"vault": name, "stats": stats})
|
||||
return {"status": "ok", "vault": name, "stats": stats}
|
||||
|
||||
|
||||
@router.delete("/api/vaults/{vault_name}", response_model=VaultActionResponse)
|
||||
async def api_remove_vault(vault_name: str, current_user=Depends(require_admin)):
|
||||
"""Remove a vault from the index and stop watching it.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault to remove.
|
||||
"""
|
||||
if vault_name not in index:
|
||||
raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found")
|
||||
|
||||
# Stop watching
|
||||
watcher = get_watcher()
|
||||
if watcher:
|
||||
await watcher.remove_vault(vault_name)
|
||||
|
||||
await remove_vault_from_index(vault_name)
|
||||
# #194 : plus de trace au redémarrage (les vaults d'env, eux, reviennent).
|
||||
from backend.indexer import unpersist_vault
|
||||
|
||||
unpersist_vault(vault_name)
|
||||
await sse_manager.broadcast("vault_removed", {"vault": vault_name})
|
||||
return {"status": "ok", "vault": vault_name}
|
||||
|
||||
|
||||
@router.get("/api/vaults/status", response_model=VaultsStatusResponse)
|
||||
async def api_vaults_status(current_user=Depends(require_auth)):
|
||||
"""Detailed status of all vaults including watcher state.
|
||||
|
||||
Returns per-vault: file count, tag count, watching status, vault path.
|
||||
"""
|
||||
watcher = get_watcher()
|
||||
statuses = {}
|
||||
for vname, vdata in index.items():
|
||||
watching = watcher is not None and vname in watcher.observers
|
||||
statuses[vname] = {
|
||||
"file_count": len(vdata.get("files", [])),
|
||||
"tag_count": len(vdata.get("tags", {})),
|
||||
"path": vdata.get("path", ""),
|
||||
"watching": watching,
|
||||
}
|
||||
return {
|
||||
"vaults": statuses,
|
||||
"watcher_active": watcher is not None,
|
||||
"sse_clients": sse_manager.client_count,
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Webhook CRUD endpoints (ROADMAP #85, tranche 2).
|
||||
|
||||
Handlers déplacés depuis :mod:`backend.main` sans changement de
|
||||
comportement : mêmes chemins (``/api/webhooks``), même modèle de réponse
|
||||
(:class:`backend.schemas.WebhookModel`), même dépendance admin. La logique
|
||||
métier vit déjà dans :mod:`backend.webhooks` (validation d'URL anti-SSRF,
|
||||
store ``webhook_secrets.json`` — BUG-026).
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
|
||||
from backend.auth.middleware import require_admin
|
||||
from backend.schemas import StatusResponse, WebhookModel
|
||||
from backend.webhooks import (
|
||||
create_webhook,
|
||||
delete_webhook,
|
||||
get_webhooks,
|
||||
update_webhook,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/webhooks", tags=["webhooks"])
|
||||
|
||||
|
||||
@router.get("", response_model=list[WebhookModel])
|
||||
async def api_webhooks_list(current_user=Depends(require_admin)):
|
||||
return get_webhooks()
|
||||
|
||||
|
||||
@router.post("", response_model=WebhookModel)
|
||||
async def api_webhooks_create(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
name = body.get("name", "Unnamed")
|
||||
url = body.get("url", "")
|
||||
events = body.get("events", [])
|
||||
secret = body.get("secret")
|
||||
if not url:
|
||||
raise HTTPException(400, "URL is required")
|
||||
# #198 : filtres optionnels vault / path_prefix (None = tout recevoir).
|
||||
return create_webhook(name, url, events, secret,
|
||||
vault=body.get("vault"), path_prefix=body.get("path_prefix"))
|
||||
|
||||
|
||||
@router.patch("/{webhook_id}", response_model=WebhookModel)
|
||||
async def api_webhooks_update(
|
||||
webhook_id: str, body: dict = Body(...), current_user=Depends(require_admin)
|
||||
):
|
||||
result = update_webhook(webhook_id, body)
|
||||
if not result:
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/{webhook_id}", response_model=StatusResponse)
|
||||
async def api_webhooks_delete(webhook_id: str, current_user=Depends(require_admin)):
|
||||
if not delete_webhook(webhook_id):
|
||||
raise HTTPException(404, "Webhook not found")
|
||||
return {"status": "deleted"}
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Scheduled tasks — automatic agent actions, type cron (#170).
|
||||
|
||||
Tasks are persisted in ``data/scheduled_tasks.json`` (guarded by an RLock,
|
||||
same pattern as the other JSON stores). Supported actions reuse the existing
|
||||
mutation/notification services — no new write path:
|
||||
|
||||
* ``create_file`` → ``backend.services.mutations.create_file``;
|
||||
* ``append_to_file`` → ``backend.services.mutations.append_to_file``;
|
||||
* ``notify`` → ``backend.notify.broadcast`` (trigger ``manual``).
|
||||
|
||||
Supported schedules:
|
||||
|
||||
* ``interval_hours`` — every N hours (N >= 0.25);
|
||||
* ``daily_time`` — once a day at ``HH:MM`` (local server time);
|
||||
* ``once_at`` — one shot at an ISO-8601 datetime (past = due immediately).
|
||||
|
||||
On failure the task records ``last_error`` and a ``schedule_failure``
|
||||
broadcast is emitted to the notification channels (#168) — best effort,
|
||||
never recursive (a failing ``notify`` action does not rebroadcast).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.scheduler")
|
||||
|
||||
DATA_DIR = Path(os.environ.get("OBSIGATE_DATA_DIR", "data"))
|
||||
TASKS_FILE = DATA_DIR / "scheduled_tasks.json"
|
||||
|
||||
ACTION_KINDS = ("create_file", "append_to_file", "notify")
|
||||
SCHEDULE_KINDS = ("interval_hours", "daily_time", "once_at")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
|
||||
# ── Store ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_tasks() -> list[dict[str, Any]]:
|
||||
if not TASKS_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(TASKS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_tasks(tasks: list[dict[str, Any]]) -> None:
|
||||
TASKS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = TASKS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(tasks, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(TASKS_FILE)
|
||||
|
||||
|
||||
def list_tasks() -> list[dict[str, Any]]:
|
||||
"""Return all scheduled tasks (newest first)."""
|
||||
return sorted(_read_tasks(), key=lambda t: t.get("created_at", ""), reverse=True)
|
||||
|
||||
|
||||
def get_task(task_id: str) -> dict[str, Any] | None:
|
||||
"""Return one task by id, or None."""
|
||||
for task in _read_tasks():
|
||||
if task.get("id") == task_id:
|
||||
return task
|
||||
return None
|
||||
|
||||
|
||||
# ── Validation ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _validate_action(action: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = action.get("kind")
|
||||
if kind not in ACTION_KINDS:
|
||||
raise ValueError(f"Action inconnue : {kind} (attendu : {', '.join(ACTION_KINDS)})")
|
||||
params = dict(action.get("params") or {})
|
||||
if kind in ("create_file", "append_to_file"):
|
||||
if not str(params.get("vault") or "").strip():
|
||||
raise ValueError("params.vault requis pour create_file/append_to_file")
|
||||
if not str(params.get("path") or "").strip():
|
||||
raise ValueError("params.path requis pour create_file/append_to_file")
|
||||
if kind == "append_to_file" and not str(params.get("content") or ""):
|
||||
raise ValueError("params.content requis pour append_to_file")
|
||||
elif kind == "notify":
|
||||
if not str(params.get("title") or "").strip():
|
||||
raise ValueError("params.title requis pour notify")
|
||||
if not str(params.get("message") or "").strip():
|
||||
raise ValueError("params.message requis pour notify")
|
||||
return {"kind": kind, "params": params}
|
||||
|
||||
|
||||
def _validate_schedule(schedule: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = schedule.get("kind")
|
||||
if kind not in SCHEDULE_KINDS:
|
||||
raise ValueError(f"Planification inconnue : {kind} (attendu : {', '.join(SCHEDULE_KINDS)})")
|
||||
if kind == "interval_hours":
|
||||
hours = float(schedule.get("hours") or 0)
|
||||
if hours < 0.25:
|
||||
raise ValueError("hours doit être >= 0.25")
|
||||
return {"kind": kind, "hours": hours}
|
||||
if kind == "daily_time":
|
||||
at = str(schedule.get("at") or "").strip()
|
||||
try:
|
||||
datetime.strptime(at, "%H:%M")
|
||||
except ValueError:
|
||||
raise ValueError("at doit être au format HH:MM (ex. 08:30)") from None
|
||||
return {"kind": kind, "at": at}
|
||||
# once_at
|
||||
at = str(schedule.get("at") or "").strip()
|
||||
try:
|
||||
parsed = datetime.fromisoformat(at)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
raise ValueError("at doit être une date ISO-8601 (ex. 2026-10-05T08:30:00)") from None
|
||||
return {"kind": kind, "at": parsed.isoformat()}
|
||||
|
||||
|
||||
# ── CRUD ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def create_task(
|
||||
name: str,
|
||||
action: dict[str, Any],
|
||||
schedule: dict[str, Any],
|
||||
*,
|
||||
created_by: str = "api",
|
||||
enabled: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a scheduled task. Raises ValueError on invalid action/schedule."""
|
||||
validated_action = _validate_action(action)
|
||||
validated_schedule = _validate_schedule(schedule)
|
||||
now = datetime.now(UTC)
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
task = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"name": (name or validated_action["kind"]).strip() or validated_action["kind"],
|
||||
"action": validated_action,
|
||||
"schedule": validated_schedule,
|
||||
"enabled": bool(enabled),
|
||||
"created_by": created_by,
|
||||
"created_at": now.isoformat(),
|
||||
"last_run_at": None,
|
||||
"last_status": None,
|
||||
"last_error": None,
|
||||
"run_count": 0,
|
||||
"next_run_at": compute_next_run(
|
||||
{"schedule": validated_schedule, "last_run_at": None}, now
|
||||
).isoformat(),
|
||||
}
|
||||
tasks.append(task)
|
||||
_write_tasks(tasks)
|
||||
logger.info(f"Scheduled task created: '{task['name']}' ({validated_schedule['kind']})")
|
||||
return task
|
||||
|
||||
|
||||
def update_task(task_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update name/enabled/action/schedule. Returns None when unknown."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
for task in tasks:
|
||||
if task.get("id") != task_id:
|
||||
continue
|
||||
if updates.get("name"):
|
||||
task["name"] = str(updates["name"])
|
||||
if "enabled" in updates:
|
||||
task["enabled"] = bool(updates["enabled"])
|
||||
if "action" in updates:
|
||||
task["action"] = _validate_action(updates["action"])
|
||||
if "schedule" in updates:
|
||||
task["schedule"] = _validate_schedule(updates["schedule"])
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
return task
|
||||
return None
|
||||
|
||||
|
||||
def delete_task(task_id: str) -> bool:
|
||||
"""Delete a task. Returns False when unknown."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
remaining = [t for t in tasks if t.get("id") != task_id]
|
||||
if len(remaining) == len(tasks):
|
||||
return False
|
||||
_write_tasks(remaining)
|
||||
return True
|
||||
|
||||
|
||||
# ── Scheduling ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def compute_next_run(task: dict[str, Any], now: datetime | None = None) -> datetime:
|
||||
"""Compute the next due datetime for *task*."""
|
||||
now = now or datetime.now(UTC)
|
||||
if now.tzinfo is None:
|
||||
now = now.replace(tzinfo=UTC)
|
||||
schedule = task.get("schedule", {})
|
||||
kind = schedule.get("kind")
|
||||
last_run_at = task.get("last_run_at")
|
||||
last = None
|
||||
if last_run_at:
|
||||
try:
|
||||
last = datetime.fromisoformat(str(last_run_at))
|
||||
if last.tzinfo is None:
|
||||
last = last.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
last = None
|
||||
if kind == "interval_hours":
|
||||
hours = float(schedule.get("hours", 24))
|
||||
base = last or now
|
||||
nxt = base + timedelta(hours=hours)
|
||||
# Première planification : due dès maintenant + intervalle ? Non —
|
||||
# la tâche démarre au prochain intervalle, sauf retard déjà accumulé.
|
||||
if last is None:
|
||||
nxt = now + timedelta(hours=hours)
|
||||
return max(now, nxt)
|
||||
if kind == "daily_time":
|
||||
hour, minute = (str(schedule.get("at", "08:00")) + ":00").split(":")[:2]
|
||||
candidate = now.replace(hour=int(hour), minute=int(minute), second=0, microsecond=0)
|
||||
if candidate <= now:
|
||||
candidate += timedelta(days=1)
|
||||
return candidate
|
||||
if kind == "once_at":
|
||||
try:
|
||||
at = datetime.fromisoformat(str(schedule.get("at")))
|
||||
if at.tzinfo is None:
|
||||
at = at.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
return now
|
||||
if task.get("last_run_at"):
|
||||
return datetime.max.replace(tzinfo=UTC) # déjà exécutée
|
||||
return at
|
||||
return now + timedelta(hours=24)
|
||||
|
||||
|
||||
def _execute_action(task: dict[str, Any]) -> dict[str, Any]:
|
||||
action = task["action"]
|
||||
kind = action["kind"]
|
||||
params = action["params"]
|
||||
if kind == "create_file":
|
||||
from backend.services.mutations import create_file
|
||||
|
||||
return create_file(
|
||||
params["vault"],
|
||||
params["path"],
|
||||
params.get("content", ""),
|
||||
overwrite=bool(params.get("overwrite", False)),
|
||||
)
|
||||
if kind == "append_to_file":
|
||||
from backend.services.mutations import append_to_file
|
||||
|
||||
return append_to_file(params["vault"], params["path"], params.get("content", ""))
|
||||
if kind == "notify":
|
||||
from backend.notify import broadcast
|
||||
|
||||
results = broadcast("manual", str(params["title"]), str(params.get("message", "")))
|
||||
return {"broadcast": results}
|
||||
raise ValueError(f"Action inconnue : {kind}")
|
||||
|
||||
|
||||
def run_task(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
||||
"""Execute one task now (manual or due). Records status; notifies on failure."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
task = next((t for t in tasks if t.get("id") == task_id), None)
|
||||
if task is None:
|
||||
raise KeyError(task_id)
|
||||
if not task.get("enabled", True) and not manual:
|
||||
return {"task_id": task_id, "skipped": True, "reason": "disabled"}
|
||||
try:
|
||||
result = _execute_action(task)
|
||||
task["last_run_at"] = datetime.now(UTC).isoformat()
|
||||
task["last_status"] = "ok"
|
||||
task["last_error"] = None
|
||||
task["run_count"] = int(task.get("run_count", 0)) + 1
|
||||
if task.get("schedule", {}).get("kind") == "once_at":
|
||||
task["enabled"] = False # one-shot consommé
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
if manual:
|
||||
from backend.notify import broadcast
|
||||
|
||||
broadcast("schedule_success", f"Tâche « {task['name']} » OK", "Exécution manuelle réussie.")
|
||||
return {"task_id": task_id, "ok": True, "result": result}
|
||||
except Exception as e:
|
||||
task["last_run_at"] = datetime.now(UTC).isoformat()
|
||||
task["last_status"] = "error"
|
||||
task["last_error"] = str(e)
|
||||
task["run_count"] = int(task.get("run_count", 0)) + 1
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
logger.warning(f"Scheduled task '{task.get('name')}' failed: {e}")
|
||||
if task["action"]["kind"] != "notify":
|
||||
try:
|
||||
from backend.notify import broadcast
|
||||
|
||||
broadcast(
|
||||
"schedule_failure",
|
||||
f"Échec tâche « {task.get('name')} »",
|
||||
f"{e}",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failure notification broadcast failed", exc_info=True)
|
||||
return {"task_id": task_id, "ok": False, "error": str(e)}
|
||||
|
||||
|
||||
def tick(now: datetime | None = None) -> list[dict[str, Any]]:
|
||||
"""Run every due task. Returns per-task outcomes (empty when idle)."""
|
||||
now = now or datetime.now(UTC)
|
||||
outcomes: list[dict[str, Any]] = []
|
||||
for task in _read_tasks():
|
||||
if not task.get("enabled", True):
|
||||
continue
|
||||
try:
|
||||
next_run = datetime.fromisoformat(str(task.get("next_run_at") or ""))
|
||||
if next_run.tzinfo is None:
|
||||
next_run = next_run.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
next_run = compute_next_run(task, now)
|
||||
if next_run <= now:
|
||||
outcomes.append(run_task(task["id"]))
|
||||
return outcomes
|
||||
+1197
File diff suppressed because it is too large
Load Diff
+312
-114
@@ -4,13 +4,25 @@ import re
|
||||
import time
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from snowballstemmer import stemmer as _snowball_stemmer
|
||||
from sortedcontainers import SortedList
|
||||
|
||||
from backend import indexer as _indexer
|
||||
from backend.indexer import index
|
||||
from backend import semantic_search as _semantic
|
||||
|
||||
# NOTE: the shared index is read through ``_indexer.index`` everywhere, never
|
||||
# via ``from backend.indexer import index``. That import binds the dict object
|
||||
# once, so a module reload of ``backend.indexer`` (tests, dev reload) rebinds
|
||||
# the module-level name to a FRESH dict while this module keeps writing to the
|
||||
# stale one — the inverted index then silently indexes nothing (BUG-089).
|
||||
from backend.services.regex_safety import (
|
||||
MAX_REGEX_MATCHES,
|
||||
truncate_for_regex,
|
||||
validate_regex,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("obsigate.search")
|
||||
|
||||
@@ -224,12 +236,16 @@ def _extract_regex_snippet(
|
||||
if not content or not pattern_text:
|
||||
return content[:200].strip() if content else ""
|
||||
|
||||
# BUG-025: bound the text scanned and the number of matches collected.
|
||||
content = truncate_for_regex(content)
|
||||
|
||||
try:
|
||||
validate_regex(pattern_text)
|
||||
pattern = re.compile(pattern_text, re.IGNORECASE)
|
||||
except re.error:
|
||||
except (re.error, ValueError):
|
||||
return _escape_html(content[:200].strip())
|
||||
|
||||
matches = list(pattern.finditer(content))
|
||||
matches = list(pattern.finditer(content))[:MAX_REGEX_MATCHES]
|
||||
if not matches:
|
||||
return _escape_html(content[:200].strip())
|
||||
|
||||
@@ -357,12 +373,19 @@ class InvertedIndex:
|
||||
self.doc_vault: dict[str, str] = {}
|
||||
self.vault_docs: dict[str, set] = defaultdict(set)
|
||||
self.tag_docs: dict[str, set] = defaultdict(set)
|
||||
self.doc_tags: dict[str, set] = defaultdict(set)
|
||||
self._sorted_tokens: SortedList = SortedList()
|
||||
self._ready: bool = False # True after initial build
|
||||
|
||||
def is_stale(self) -> bool:
|
||||
"""Return True if the index has not been built yet."""
|
||||
return not self._ready
|
||||
def is_ready(self) -> bool:
|
||||
"""Return True once the initial build has completed.
|
||||
|
||||
The index is then kept current incrementally by ``add_document()`` /
|
||||
``remove_document()``, so it never goes stale: there is no generation
|
||||
counter, no cooldown and no lazy rebuild. Searches simply fall back to
|
||||
a full scan while this is False (see ``search()``).
|
||||
"""
|
||||
return self._ready
|
||||
|
||||
def rebuild(self) -> None:
|
||||
"""Rebuild inverted index from the global ``index`` dict.
|
||||
@@ -381,8 +404,9 @@ class InvertedIndex:
|
||||
self.doc_vault = {}
|
||||
self.vault_docs = defaultdict(set)
|
||||
self.tag_docs = defaultdict(set)
|
||||
self.doc_tags = defaultdict(set)
|
||||
|
||||
for vault_name, vault_data in index.items():
|
||||
for vault_name, vault_data in _indexer.index.items():
|
||||
for file_info in vault_data.get("files", []):
|
||||
doc_key = f"{vault_name}::{file_info['path']}"
|
||||
self.doc_count += 1
|
||||
@@ -395,6 +419,7 @@ class InvertedIndex:
|
||||
# --- Per-document tag index ---
|
||||
for tag in file_info.get("tags", []):
|
||||
self.tag_docs[tag.lower()].add(doc_key)
|
||||
self.doc_tags[file_info['path']].add(tag.lower())
|
||||
|
||||
# --- Title tokens ---
|
||||
title_tokens = tokenize(file_info.get("title", ""))
|
||||
@@ -526,6 +551,10 @@ class InvertedIndex:
|
||||
self.doc_vault.pop(doc_key, None)
|
||||
if vault_name in self.vault_docs:
|
||||
self.vault_docs[vault_name].discard(doc_key)
|
||||
# Drop the empty entry so a fully removed vault leaves no trace
|
||||
# (it is a defaultdict: a bare lookup would recreate the key).
|
||||
if not self.vault_docs[vault_name]:
|
||||
del self.vault_docs[vault_name]
|
||||
# Tags (per-document, NOT the global tag_norm_map)
|
||||
for tag in file_info.get("tags", []):
|
||||
td = self.tag_docs.get(tag.lower())
|
||||
@@ -655,6 +684,10 @@ def _on_index_change_hook(action: str, vault_name: str, path: str, file_info: di
|
||||
inv.remove_document(vault_name, path)
|
||||
except Exception as e:
|
||||
logger.warning(f"Inverted index incremental update failed ({action} {vault_name}/{path}): {e}")
|
||||
try:
|
||||
_semantic.on_index_change(action, vault_name, path, file_info)
|
||||
except Exception as e:
|
||||
logger.warning(f"Semantic index incremental update failed ({action} {vault_name}/{path}): {e}")
|
||||
|
||||
|
||||
# Register the hook with indexer (indexer is already imported at top of file)
|
||||
@@ -663,7 +696,7 @@ _indexer.set_index_change_hook(_on_index_change_hook)
|
||||
|
||||
def init_inverted_index():
|
||||
"""Force initial inverted index build. Called after build_index completes on startup."""
|
||||
if any(vdata.get("files") for vdata in index.values()):
|
||||
if any(vdata.get("files") for vdata in _indexer.index.values()):
|
||||
_inverted_index.rebuild()
|
||||
logger.info("Inverted index initialized.")
|
||||
|
||||
@@ -723,60 +756,97 @@ def search(
|
||||
query_lower = query.lower()
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for vault_name, vault_data in index.items():
|
||||
if vault_filter != "all" and vault_name != vault_filter:
|
||||
inv = get_inverted_index()
|
||||
use_index = inv.is_ready() and inv.doc_count > 0
|
||||
|
||||
if use_index:
|
||||
# BUG-033: retrieve candidates from the inverted index instead of
|
||||
# scanning every document. Multi-term queries require all terms
|
||||
# (a superset of exact-phrase matches), single terms use prefix
|
||||
# expansion. Falls back to a full scan while the index is building.
|
||||
if has_query:
|
||||
terms = [t for t in tokenize(query) if t]
|
||||
if not terms:
|
||||
return []
|
||||
doc_sets: list[set] = []
|
||||
for term in terms:
|
||||
term_docs: set = set(inv.word_index.get(term, {}).keys())
|
||||
if len(term) >= MIN_PREFIX_LENGTH:
|
||||
for expanded in inv.get_prefix_tokens(term):
|
||||
term_docs.update(inv.word_index.get(expanded, {}).keys())
|
||||
doc_sets.append(term_docs)
|
||||
doc_keys = set.intersection(*doc_sets) if doc_sets else set()
|
||||
else:
|
||||
doc_keys = set(inv.doc_info.keys())
|
||||
|
||||
if vault_filter != "all":
|
||||
doc_keys &= inv.vault_docs.get(vault_filter, set())
|
||||
for tag in selected_tags:
|
||||
doc_keys &= inv.tag_docs.get(tag.lower(), set())
|
||||
|
||||
candidates = [
|
||||
(inv.doc_vault[dk], inv.doc_info[dk])
|
||||
for dk in doc_keys
|
||||
if dk in inv.doc_info
|
||||
]
|
||||
else:
|
||||
candidates = [
|
||||
(vault_name, file_info)
|
||||
for vault_name, vault_data in _indexer.index.items()
|
||||
if vault_filter == "all" or vault_name == vault_filter
|
||||
for file_info in vault_data["files"]
|
||||
]
|
||||
|
||||
for vault_name, file_info in candidates:
|
||||
# Tag filter: all selected tags must be present
|
||||
if selected_tags and not all(tag in file_info["tags"] for tag in selected_tags):
|
||||
continue
|
||||
|
||||
for file_info in vault_data["files"]:
|
||||
# Tag filter: all selected tags must be present
|
||||
if selected_tags and not all(tag in file_info["tags"] for tag in selected_tags):
|
||||
continue
|
||||
score = 0
|
||||
snippet = file_info.get("content_preview", "")
|
||||
|
||||
score = 0
|
||||
snippet = file_info.get("content_preview", "")
|
||||
if has_query:
|
||||
title_lower = file_info["title"].lower()
|
||||
|
||||
if has_query:
|
||||
title_lower = file_info["title"].lower()
|
||||
# Exact title match (highest weight)
|
||||
if query_lower == title_lower:
|
||||
score += 20
|
||||
# Partial title match
|
||||
elif query_lower in title_lower:
|
||||
score += 10
|
||||
|
||||
# Exact title match (highest weight)
|
||||
if query_lower == title_lower:
|
||||
score += 20
|
||||
# Partial title match
|
||||
elif query_lower in title_lower:
|
||||
score += 10
|
||||
# Path match (folder/filename relevance)
|
||||
if query_lower in file_info["path"].lower():
|
||||
score += 5
|
||||
|
||||
# Path match (folder/filename relevance)
|
||||
if query_lower in file_info["path"].lower():
|
||||
score += 5
|
||||
# Tag name match
|
||||
for tag in file_info.get("tags", []):
|
||||
if query_lower in tag.lower():
|
||||
score += 3
|
||||
break # count once per file
|
||||
|
||||
# Tag name match
|
||||
for tag in file_info.get("tags", []):
|
||||
if query_lower in tag.lower():
|
||||
score += 3
|
||||
break # count once per file
|
||||
# Content match — use cached content (no disk I/O)
|
||||
content = file_info.get("content", "")
|
||||
content_lower = content.lower()
|
||||
if query_lower in content_lower:
|
||||
# Frequency-based scoring, capped to avoid over-weighting
|
||||
occurrences = content_lower.count(query_lower)
|
||||
score += min(occurrences, 10)
|
||||
snippet = _extract_snippet(content, query)
|
||||
else:
|
||||
# Tag-only filter: all matching files get score 1
|
||||
score = 1
|
||||
|
||||
# Content match — use cached content (no disk I/O)
|
||||
content = file_info.get("content", "")
|
||||
content_lower = content.lower()
|
||||
if query_lower in content_lower:
|
||||
# Frequency-based scoring, capped to avoid over-weighting
|
||||
occurrences = content_lower.count(query_lower)
|
||||
score += min(occurrences, 10)
|
||||
snippet = _extract_snippet(content, query)
|
||||
else:
|
||||
# Tag-only filter: all matching files get score 1
|
||||
score = 1
|
||||
|
||||
if score > 0:
|
||||
results.append({
|
||||
"vault": vault_name,
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info["tags"],
|
||||
"score": score,
|
||||
"snippet": snippet,
|
||||
"modified": file_info["modified"],
|
||||
})
|
||||
if score > 0:
|
||||
results.append({
|
||||
"vault": vault_name,
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info["tags"],
|
||||
"score": score,
|
||||
"snippet": snippet,
|
||||
"modified": file_info["modified"],
|
||||
})
|
||||
|
||||
results.sort(key=lambda x: -x["score"])
|
||||
return results[:limit]
|
||||
@@ -902,12 +972,14 @@ def _passes_search_filters(
|
||||
title = file_info.get("title", "")
|
||||
content = file_info.get("content", "")
|
||||
path = file_info.get("path", "")
|
||||
search_text = f"{title} {content}"
|
||||
# BUG-025: cap the text scanned by a user-supplied regex.
|
||||
search_text = truncate_for_regex(f"{title} {content}")
|
||||
search_text_norm = normalize_text(search_text)
|
||||
|
||||
# --- Regex mode ---
|
||||
if regex and raw_query:
|
||||
try:
|
||||
validate_regex(raw_query)
|
||||
flags = 0 if case_sensitive else re.IGNORECASE
|
||||
if whole_word:
|
||||
pattern = re.compile(rf"\b{raw_query}\b", flags)
|
||||
@@ -915,7 +987,7 @@ def _passes_search_filters(
|
||||
pattern = re.compile(raw_query, flags)
|
||||
if not pattern.search(search_text):
|
||||
return False
|
||||
except re.error:
|
||||
except (re.error, ValueError):
|
||||
return False
|
||||
return _passes_path_filters(path, include_paths, exclude_paths)
|
||||
|
||||
@@ -1102,6 +1174,7 @@ def advanced_search(
|
||||
created: str | None = None,
|
||||
modified: str | None = None,
|
||||
size: str | None = None,
|
||||
semantic: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Advanced full-text search with TF-IDF scoring, facets, and pagination.
|
||||
|
||||
@@ -1121,13 +1194,21 @@ def advanced_search(
|
||||
limit: Max results per page.
|
||||
offset: Pagination offset.
|
||||
sort_by: ``"relevance"`` or ``"modified"``.
|
||||
semantic: When True, fuse the TF-IDF ranking with the semantic
|
||||
(embedding) ranking via Reciprocal Rank Fusion and expose a
|
||||
``semantic_score`` per result.
|
||||
|
||||
Returns:
|
||||
Dict with ``results``, ``total``, ``offset``, ``limit``, ``facets``,
|
||||
``query_time_ms``.
|
||||
``query_time_ms`` and ``semantic_available``.
|
||||
"""
|
||||
t0 = time.monotonic()
|
||||
query = query.strip() if query else ""
|
||||
|
||||
# BUG-025: reject oversized / catastrophic regex patterns up front.
|
||||
if regex and query:
|
||||
validate_regex(query)
|
||||
|
||||
parsed = _parse_advanced_query(query)
|
||||
|
||||
# Merge explicit tag_filter with parsed tag: operators
|
||||
@@ -1182,58 +1263,63 @@ def advanced_search(
|
||||
# ------------------------------------------------------------------
|
||||
# Step 2: Apply filters on candidate set
|
||||
# ------------------------------------------------------------------
|
||||
if effective_vault != "all":
|
||||
candidates &= inv.vault_docs.get(effective_vault, set())
|
||||
|
||||
if all_tags and has_terms:
|
||||
for t in all_tags:
|
||||
candidates &= inv.tag_docs.get(t.lower(), set())
|
||||
|
||||
if parsed["title"]:
|
||||
norm_title_filter = normalize_text(parsed["title"])
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if norm_title_filter in normalize_text(inv.doc_info[dk].get("title", ""))
|
||||
}
|
||||
|
||||
if parsed["path"]:
|
||||
norm_path_filter = normalize_text(parsed["path"])
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if norm_path_filter in normalize_text(inv.doc_info[dk].get("path", ""))
|
||||
}
|
||||
|
||||
if parsed["ext"]:
|
||||
ext_filter = parsed["ext"]
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if (
|
||||
inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower() == ext_filter
|
||||
or inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower().endswith(f".{ext_filter}")
|
||||
)
|
||||
}
|
||||
|
||||
# Date and size filters (from query operators or API params)
|
||||
date_range_created = _parse_date_range(created or parsed.get("created"))
|
||||
if date_range_created:
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if _matches_date_range(inv.doc_info[dk].get("created"), date_range_created)
|
||||
}
|
||||
|
||||
date_range_modified = _parse_date_range(modified or parsed.get("modified"))
|
||||
if date_range_modified:
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if _matches_date_range(inv.doc_info[dk].get("modified"), date_range_modified)
|
||||
}
|
||||
|
||||
size_range = _parse_size_range(size or parsed.get("size"))
|
||||
if size_range:
|
||||
candidates = {
|
||||
dk for dk in candidates
|
||||
if _matches_size_range(inv.doc_info[dk].get("size", 0), size_range)
|
||||
}
|
||||
|
||||
def _apply_metadata_filters(docs: set) -> set:
|
||||
"""Restrict a document-key set to the query's metadata filters."""
|
||||
if effective_vault != "all":
|
||||
docs &= inv.vault_docs.get(effective_vault, set())
|
||||
|
||||
if all_tags:
|
||||
for t in all_tags:
|
||||
docs &= inv.tag_docs.get(t.lower(), set())
|
||||
|
||||
if parsed["title"]:
|
||||
norm_title_filter = normalize_text(parsed["title"])
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if norm_title_filter in normalize_text(inv.doc_info[dk].get("title", ""))
|
||||
}
|
||||
|
||||
if parsed["path"]:
|
||||
norm_path_filter = normalize_text(parsed["path"])
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if norm_path_filter in normalize_text(inv.doc_info[dk].get("path", ""))
|
||||
}
|
||||
|
||||
if parsed["ext"]:
|
||||
ext_filter = parsed["ext"]
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if (
|
||||
inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower() == ext_filter
|
||||
or inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower().endswith(f".{ext_filter}")
|
||||
)
|
||||
}
|
||||
|
||||
if date_range_created:
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if _matches_date_range(inv.doc_info[dk].get("created"), date_range_created)
|
||||
}
|
||||
|
||||
if date_range_modified:
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if _matches_date_range(inv.doc_info[dk].get("modified"), date_range_modified)
|
||||
}
|
||||
|
||||
if size_range:
|
||||
docs = {
|
||||
dk for dk in docs
|
||||
if _matches_size_range(inv.doc_info[dk].get("size", 0), size_range)
|
||||
}
|
||||
return docs
|
||||
|
||||
candidates = _apply_metadata_filters(candidates)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step 3: Score only the candidates (not all N documents)
|
||||
@@ -1241,6 +1327,7 @@ def advanced_search(
|
||||
scored_results: list[tuple[float, dict[str, Any]]] = []
|
||||
facet_tags: dict[str, int] = defaultdict(int)
|
||||
facet_vaults: dict[str, int] = defaultdict(int)
|
||||
facet_extensions: dict[str, int] = defaultdict(int)
|
||||
|
||||
# Pre-compute prefix expansions once per term (avoid repeated binary search)
|
||||
prefix_expansions: dict[str, list[str]] = {}
|
||||
@@ -1312,16 +1399,22 @@ def advanced_search(
|
||||
"title": file_info["title"],
|
||||
"tags": file_info.get("tags", []),
|
||||
"score": round(score, 4),
|
||||
"semantic_score": 0.0,
|
||||
"snippet": snippet,
|
||||
"modified": file_info.get("modified", ""),
|
||||
"extension": file_info.get("extension", file_info.get("path", "").rsplit(".", 1)[-1] if "." in file_info.get("path", "") else ""),
|
||||
}
|
||||
scored_results.append((score, result))
|
||||
|
||||
# Facets
|
||||
facet_vaults[vault_name] = facet_vaults.get(vault_name, 0) + 1
|
||||
for tag in file_info.get("tags", []):
|
||||
facet_tags[tag] = facet_tags.get(tag, 0) + 1
|
||||
# ------------------------------------------------------------------
|
||||
# Step 4: Optional semantic fusion (RRF with the TF-IDF ranking)
|
||||
# ------------------------------------------------------------------
|
||||
semantic_available = _semantic.get_semantic_index().is_ready()
|
||||
if semantic and has_terms and not regex:
|
||||
scored_results = _fuse_semantic_results(
|
||||
scored_results, query, effective_vault, inv, _apply_metadata_filters,
|
||||
include_paths, exclude_paths, limit,
|
||||
)
|
||||
|
||||
# Sort
|
||||
if sort_by == "modified":
|
||||
@@ -1329,6 +1422,17 @@ def advanced_search(
|
||||
else:
|
||||
scored_results.sort(key=lambda x: -x[0])
|
||||
|
||||
# Facets are recomputed from the final result set (covers semantic-only docs)
|
||||
for _, result in scored_results:
|
||||
facet_vaults[result["vault"]] = facet_vaults.get(result["vault"], 0) + 1
|
||||
for tag in result.get("tags", []):
|
||||
facet_tags[tag] = facet_tags.get(tag, 0) + 1
|
||||
# Extension normalised without leading dot ("md", not ".md") so it can be
|
||||
# fed back directly as the `ext:` query operator.
|
||||
ext = str(result.get("extension") or "").lower().lstrip(".")
|
||||
if ext:
|
||||
facet_extensions[ext] = facet_extensions.get(ext, 0) + 1
|
||||
|
||||
total = len(scored_results)
|
||||
page = scored_results[offset: offset + limit]
|
||||
elapsed_ms = round((time.monotonic() - t0) * 1000, 1)
|
||||
@@ -1341,11 +1445,102 @@ def advanced_search(
|
||||
"facets": {
|
||||
"tags": dict(sorted(facet_tags.items(), key=lambda x: -x[1])[:20]),
|
||||
"vaults": dict(sorted(facet_vaults.items(), key=lambda x: -x[1])),
|
||||
"extensions": dict(sorted(facet_extensions.items(), key=lambda x: -x[1])),
|
||||
},
|
||||
"query_time_ms": elapsed_ms,
|
||||
"semantic_available": semantic_available,
|
||||
}
|
||||
|
||||
|
||||
def _fuse_semantic_results(
|
||||
scored_results: list[tuple[float, dict[str, Any]]],
|
||||
query: str,
|
||||
vault_filter: str,
|
||||
inv: InvertedIndex,
|
||||
apply_metadata_filters: Callable[[set], set],
|
||||
include_paths: str | None,
|
||||
exclude_paths: str | None,
|
||||
limit: int,
|
||||
) -> list[tuple[float, dict[str, Any]]]:
|
||||
"""Fuse the TF-IDF ranking with the semantic ranking using RRF.
|
||||
|
||||
Documents found only by the semantic engine are materialized from the
|
||||
inverted index metadata (with a plain, non-highlighted snippet). The
|
||||
returned tuples carry the fused score, and every result dict gets a
|
||||
``semantic_score`` (cosine similarity, 0.0 when absent).
|
||||
|
||||
Args:
|
||||
scored_results: Existing ``(tfidf_score, result_dict)`` tuples.
|
||||
query: Raw free-text query.
|
||||
vault_filter: Effective vault filter.
|
||||
inv: Inverted index (document metadata source).
|
||||
apply_metadata_filters: Callable restricting a doc-key set to the
|
||||
query's tag/title/path/ext/date/size filters.
|
||||
include_paths: Include glob patterns (or None).
|
||||
exclude_paths: Exclude glob patterns (or None).
|
||||
limit: Requested page size (drives how many semantic hits to fetch).
|
||||
|
||||
Returns:
|
||||
New ``(fused_score, result_dict)`` list (unsorted).
|
||||
"""
|
||||
sem_hits = _semantic.semantic_search_docs(query, vault_filter=vault_filter, top_k=max(limit * 5, 200))
|
||||
if not sem_hits:
|
||||
return scored_results
|
||||
|
||||
# Semantic candidates must satisfy the same metadata + path filters.
|
||||
semantic_universe = apply_metadata_filters(set(inv.doc_info.keys()))
|
||||
semantic_universe = {
|
||||
dk for dk in semantic_universe
|
||||
if _passes_path_filters(inv.doc_info[dk].get("path", ""), include_paths, exclude_paths)
|
||||
}
|
||||
|
||||
sem_scores: dict[str, float] = {}
|
||||
sem_ranked: list[str] = []
|
||||
for doc_key, similarity in sem_hits:
|
||||
if doc_key not in semantic_universe:
|
||||
continue
|
||||
sem_scores[doc_key] = similarity
|
||||
sem_ranked.append(doc_key)
|
||||
|
||||
if not sem_ranked:
|
||||
return scored_results
|
||||
|
||||
existing: dict[str, dict[str, Any]] = {}
|
||||
lexical_ranked: list[str] = []
|
||||
for _, lex_result in sorted(scored_results, key=lambda item: -item[0]):
|
||||
key = f"{lex_result['vault']}::{lex_result['path']}"
|
||||
existing[key] = lex_result
|
||||
lexical_ranked.append(key)
|
||||
|
||||
fused = _semantic.rrf_fuse([lexical_ranked, sem_ranked])
|
||||
|
||||
merged: list[tuple[float, dict[str, Any]]] = []
|
||||
for doc_key, fused_score in fused.items():
|
||||
result = existing.get(doc_key)
|
||||
if result is None:
|
||||
file_info = inv.doc_info.get(doc_key)
|
||||
if file_info is None:
|
||||
continue
|
||||
content = file_info.get("content", "")
|
||||
result = {
|
||||
"vault": inv.doc_vault[doc_key],
|
||||
"path": file_info["path"],
|
||||
"title": file_info["title"],
|
||||
"tags": file_info.get("tags", []),
|
||||
"score": 0.0,
|
||||
"semantic_score": 0.0,
|
||||
"snippet": _escape_html(content[:200].strip()) if content else "",
|
||||
"modified": file_info.get("modified", ""),
|
||||
"extension": file_info.get(
|
||||
"extension",
|
||||
file_info.get("path", "").rsplit(".", 1)[-1] if "." in file_info.get("path", "") else "",
|
||||
),
|
||||
}
|
||||
result["semantic_score"] = round(sem_scores.get(doc_key, 0.0), 4)
|
||||
merged.append((fused_score, result))
|
||||
return merged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Suggestion helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1353,7 +1548,7 @@ def suggest_titles(
|
||||
prefix: str,
|
||||
vault_filter: str = "all",
|
||||
limit: int = SUGGEST_LIMIT,
|
||||
) -> list[dict[str, str]]:
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Suggest file titles matching a prefix (accent-insensitive).
|
||||
|
||||
Args:
|
||||
@@ -1362,7 +1557,7 @@ def suggest_titles(
|
||||
limit: Maximum suggestions.
|
||||
|
||||
Returns:
|
||||
List of ``{"vault", "path", "title"}`` dicts.
|
||||
List of ``{"vault", "path", "title", "tags"}`` dicts.
|
||||
"""
|
||||
if not prefix or len(prefix) < MIN_PREFIX_LENGTH:
|
||||
return []
|
||||
@@ -1380,7 +1575,10 @@ def suggest_titles(
|
||||
key = f"{entry['vault']}::{entry['path']}"
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
results.append(entry)
|
||||
# Add tags from the index
|
||||
entry_with_tags: dict[str, Any] = dict(entry)
|
||||
entry_with_tags["tags"] = list(inv.doc_tags.get(entry["path"], set()))
|
||||
results.append(entry_with_tags)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
|
||||
@@ -1433,7 +1631,7 @@ def get_all_tags(vault_filter: str | None = None) -> dict[str, int]:
|
||||
Dict mapping tag names to their total occurrence count.
|
||||
"""
|
||||
merged: dict[str, int] = {}
|
||||
for vault_name, vault_data in index.items():
|
||||
for vault_name, vault_data in _indexer.index.items():
|
||||
if vault_filter and vault_filter != "all" and vault_name != vault_filter:
|
||||
continue
|
||||
for tag, count in vault_data.get("tags", {}).items():
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Shared thread pool for CPU-bound search (ROADMAP #85, tranche 5).
|
||||
|
||||
Holder extrait de :mod:`backend.main` sans changement de comportement :
|
||||
un seul pool (2 workers, préfixe ``"search"``) créé au démarrage et arrêté
|
||||
à l'extinction par le lifespan de ``main``. Les routers et les endpoints
|
||||
restants y accèdent via :func:`get_search_executor` au lieu du global de
|
||||
``main`` (plus d'import circulaire potentiel).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
_executor: ThreadPoolExecutor | None = None
|
||||
|
||||
|
||||
def init_search_executor(max_workers: int = 2) -> ThreadPoolExecutor:
|
||||
"""Create (or reuse) the shared search thread pool."""
|
||||
global _executor
|
||||
if _executor is None:
|
||||
_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="search")
|
||||
return _executor
|
||||
|
||||
|
||||
def shutdown_search_executor() -> None:
|
||||
"""Stop the shared search thread pool (best-effort, non-blocking)."""
|
||||
global _executor
|
||||
if _executor is not None:
|
||||
_executor.shutdown(wait=False)
|
||||
_executor = None
|
||||
|
||||
|
||||
def get_search_executor() -> ThreadPoolExecutor | None:
|
||||
"""Return the shared search thread pool (``None`` before startup)."""
|
||||
return _executor
|
||||
+231
-36
@@ -1,52 +1,183 @@
|
||||
"""
|
||||
Secret redactor: masks sensitive patterns in rendered text.
|
||||
|
||||
Scans for common secret patterns and replaces them with [MASQUÉ]
|
||||
before content is served to the frontend. Prevents accidental
|
||||
exposure of API keys, tokens, and passwords in previews.
|
||||
Scans for common secret patterns and replaces them with a French mask
|
||||
label (``[CLÉ API MASQUÉE]``, ``[MOT DE PASSE MASQUÉ]``, …) before content
|
||||
is served to the frontend. Prevents accidental exposure of API keys,
|
||||
tokens, and passwords in previews.
|
||||
|
||||
Patterns detected:
|
||||
- Generic API keys (long alphanumeric strings with key/secret/token prefix)
|
||||
- Generic API keys (``api_key=…``, ``token: …`` — values of 8+ chars)
|
||||
- Passwords (``password=…``, ``"passwd": "…"`` — any length)
|
||||
- JWT tokens (eyJ... base64url)
|
||||
- AWS-style keys (AKIA..., sk-..., etc.)
|
||||
- Provider key formats: OpenAI/Anthropic/OpenRouter (``sk-``), Stripe,
|
||||
GitLab, Google (``AIza…`` / ``ya29.``), AWS, GitHub, Slack, SendGrid,
|
||||
Hugging Face, npm, Docker, Resend, Square, Atlassian, Discord,
|
||||
Telegram, ``Bearer …`` tokens
|
||||
- Private key blocks (-----BEGIN ... PRIVATE KEY-----)
|
||||
- Connection strings with passwords
|
||||
- Bare hex secrets next to a secret keyword (BUG-035)
|
||||
|
||||
Interactive masking (feature #188): :func:`redact_with_placeholders`
|
||||
returns the text with every mask replaced by an opaque placeholder plus
|
||||
the list of ``(label, secret)`` entries; :func:`restore_masks` turns the
|
||||
placeholders back into plain labels (public shares, PDF exports, AI
|
||||
context) or into clickable ``<span class="secret-mask" data-secret="…">``
|
||||
badges (authenticated app preview) so a click copies the real value to
|
||||
the clipboard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as _html
|
||||
import logging
|
||||
import re
|
||||
|
||||
logger = logging.getLogger("obsigate.redactor")
|
||||
|
||||
# --- Patterns ---
|
||||
# Order matters: more specific patterns first
|
||||
_PATTERNS = [
|
||||
# Each entry is ``(pattern, replacement, secret_group)``:
|
||||
# * ``replacement``: a literal label, a ``\\1``-style template, or a
|
||||
# callable receiving the match and returning the visible label;
|
||||
# * ``secret_group``: index of the group holding the value that a click
|
||||
# copies to the clipboard (feature #188).
|
||||
_PATTERNS: list[tuple[re.Pattern[str], object, int]] = [
|
||||
# Private key blocks
|
||||
(re.compile(r'-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----.*?-----END (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----', re.DOTALL), '[CLÉ PRIVÉE MASQUÉE]'),
|
||||
|
||||
(re.compile(r'-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----.*?-----END (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----', re.DOTALL), '[CLÉ PRIVÉE MASQUÉE]', 0),
|
||||
|
||||
# JWT tokens (base64url encoded, starts with eyJ)
|
||||
(re.compile(r'eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}'), '[JWT MASQUÉ]'),
|
||||
|
||||
(re.compile(r'eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}'), '[JWT MASQUÉ]', 0),
|
||||
|
||||
# Connection strings with passwords
|
||||
(re.compile(r'(?:mongodb|mysql|postgres(?:ql)?|redis|sqlite)://[^:]+:[^@\s]+@'), '[CONNECTION_STRING MASQUÉE]'),
|
||||
|
||||
# Generic API key patterns: key=... or token=... or secret=...
|
||||
(re.compile(r'(?:api[_-]?key|apikey|secret|token|password|passwd|auth[_-]?token)\s*[:=]\s*[\'"]?([^\s\'"]{20,})[\'"]?', re.IGNORECASE),
|
||||
lambda m: f'{m.group(0).split("=")[0].split(":")[0]}=[MASQUÉ]' if "=" in m.group(0) or ":" in m.group(0) else '[MASQUÉ]'),
|
||||
|
||||
# Generic long hex/base64 strings that look like secrets (40+ chars)
|
||||
(re.compile(r'(?:sk|pk|rk)-[a-zA-Z0-9]{20,}'), '[CLÉ API MASQUÉE]'),
|
||||
|
||||
# AWS access keys
|
||||
(re.compile(r'AKIA[0-9A-Z]{16}'), '[AWS_KEY MASQUÉ]'),
|
||||
|
||||
(re.compile(r'(?:mongodb|mysql|postgres(?:ql)?|redis|sqlite)://[^:]+:[^@\s]+@'), '[CONNECTION_STRING MASQUÉE]', 0),
|
||||
|
||||
# Passwords — any length, bare or quoted (``password=…``,
|
||||
# ``"passwd": "…"``). The left side may carry a qualifier
|
||||
# (``db_password``, ``DATABASE.PASSWORD``, ``user_pwd``); a *bare*
|
||||
# ``PWD=`` (shell working directory) must NOT match, hence ``pwd``
|
||||
# only in its qualified branch.
|
||||
(re.compile(
|
||||
r'(?i)((?:[A-Za-z0-9_.-]*(?:password|passwd|passphrase|mot\s+de\s+passe)'
|
||||
r'|[A-Za-z0-9_.-]+pwd)["\']?\s*[:=]\s*["\']?)([^\s"\',;]{4,})'),
|
||||
r'\1[MOT DE PASSE MASQUÉ]', 2),
|
||||
|
||||
# Generic API key assignments: api_key=…, token=…, secret=… — values
|
||||
# of 8+ characters (short enough to catch real keys, long enough to
|
||||
# skip plain words).
|
||||
(re.compile(r'(?i)([A-Za-z0-9_.-]*(?:api[_-]?key|apikey|secret|token|auth[_-]?token)["\']?\s*[:=]\s*["\']?)([^\s\'"]{8,})'),
|
||||
lambda m: f'{m.group(1)}[MASQUÉ]' if ("=" in m.group(0) or ":" in m.group(0)) else '[MASQUÉ]', 2),
|
||||
|
||||
# GitHub tokens (ghp_, gho_, ghu_, ghs_, ghr_)
|
||||
(re.compile(r'gh[pousr]_[a-zA-Z0-9]{36,}'), '[GITHUB_TOKEN MASQUÉ]'),
|
||||
|
||||
# Generic long random-looking strings (40+ hex chars)
|
||||
(re.compile(r'\b[a-fA-F0-9]{40,64}\b'), '[HEX_KEY MASQUÉ]'),
|
||||
(re.compile(r'gh[pousr]_[a-zA-Z0-9]{36,}'), '[GITHUB_TOKEN MASQUÉ]', 0),
|
||||
|
||||
# AWS access keys
|
||||
(re.compile(r'(?:AKIA|ASIA)[0-9A-Z]{16}'), '[AWS_KEY MASQUÉ]', 0),
|
||||
|
||||
# Provider key formats (feature #188) — one alternation covering the
|
||||
# large majority of token shapes in the wild.
|
||||
(re.compile(
|
||||
r'(?<![A-Za-z0-9])(?:'
|
||||
r'sk-[A-Za-z0-9_\-]{16,}' # OpenAI / Anthropic / OpenRouter
|
||||
r'|sk_(?:live|test)_[A-Za-z0-9]{10,}' # Stripe secret key
|
||||
r'|pk_(?:live|test)_[A-Za-z0-9]{10,}' # Stripe publishable key
|
||||
r'|whsec_[A-Za-z0-9]{16,}' # Stripe / Svix webhook secret
|
||||
r'|glpat-[A-Za-z0-9_\-]{20,}' # GitLab personal access token
|
||||
r'|github_pat_[A-Za-z0-9_]{22,}' # GitHub fine-grained PAT
|
||||
r'|npm_[A-Za-z0-9]{36}' # npm automation token
|
||||
r'|dckr_pat_[A-Za-z0-9_\-]{20,}' # Docker Hub token
|
||||
r'|hf_[A-Za-z0-9]{30,}' # Hugging Face token
|
||||
r'|AIza[0-9A-Za-z_\-]{35}' # Google API key
|
||||
r'|ya29\.[0-9A-Za-z_\-]{20,}' # Google OAuth access token
|
||||
r'|xox[baprs]-[0-9A-Za-z\-]{10,}' # Slack token
|
||||
r'|SG\.[A-Za-z0-9_\-]{16,}' # SendGrid API key
|
||||
r'|re_[A-Za-z0-9]{40}' # Resend API key
|
||||
r'|sq0[a-z]{3}-[A-Za-z0-9_\-]{16,}' # Square access token
|
||||
r'|ATATT[A-Za-z0-9_\-]{20,}' # Atlassian access token
|
||||
r'|[NOP][A-Za-z0-9_\-]{23,28}\.[A-Za-z0-9_\-]{6}\.[A-Za-z0-9_\-]{27,}' # Discord bot token
|
||||
r'|\d{8,10}:[A-Za-z0-9_\-]{35}' # Telegram bot token
|
||||
r'|Bearer\s+[A-Za-z0-9._~+/=\-]{20,}' # Authorization: Bearer …
|
||||
r')'),
|
||||
'[CLÉ API MASQUÉE]', 0),
|
||||
|
||||
]
|
||||
|
||||
# BUG-035: bare 40–64 char hex strings used to be redacted unconditionally,
|
||||
# which mangled legitimate git commit SHAs, checksums and hashes in notes.
|
||||
# They are now only redacted when a secret-ish keyword sits in the immediate
|
||||
# context; hash/commit keywords explicitly exempt them.
|
||||
_HEX_RE = re.compile(r'\b[a-fA-F0-9]{40,64}\b')
|
||||
_SECRET_CONTEXT_RE = re.compile(
|
||||
r'(?i)\b(?:secret|token|key|apikey|api[_-]?key|password|passwd|auth|bearer|'
|
||||
r'credential|x-api-key|x-auth-token)\b'
|
||||
)
|
||||
_HASH_CONTEXT_RE = re.compile(
|
||||
r'(?i)\b(?:commit|sha\d*|hash|md5|blob|git|checksum|digest|integrity|'
|
||||
r'revision|rev|etag|fingerprint)\b'
|
||||
)
|
||||
#: How far before the hex string a keyword may appear to count as context.
|
||||
_HEX_CONTEXT_WINDOW = 60
|
||||
|
||||
# --- Interactive masking (feature #188) ---
|
||||
# Private-use-area sentinels: they survive markdown rendering (mistune
|
||||
# treats them as plain text, fenced code blocks included) and are
|
||||
# stripped by ``backend.render._heading_slugify``.
|
||||
_PLACEHOLDER_OPEN = "\uE000"
|
||||
_PLACEHOLDER_CLOSE = "\uE001"
|
||||
_PLACEHOLDER_RE = re.compile("\uE000(\\d+)\uE001")
|
||||
|
||||
|
||||
def _redact_bare_hex_secrets(text: str, mask) -> tuple[str, int]:
|
||||
"""Redact 40–64 char hex strings only when a secret keyword is nearby.
|
||||
|
||||
Git/SHA/checksum contexts are left untouched (BUG-035).
|
||||
|
||||
Args:
|
||||
text: Text to scan.
|
||||
mask: ``mask(original, label) -> str`` replacement builder.
|
||||
|
||||
Returns:
|
||||
(redacted_text, redaction_count) tuple.
|
||||
"""
|
||||
count = 0
|
||||
|
||||
def _replace(match: re.Match) -> str:
|
||||
nonlocal count
|
||||
window = text[max(0, match.start() - _HEX_CONTEXT_WINDOW):match.start()]
|
||||
if _HASH_CONTEXT_RE.search(window):
|
||||
return match.group(0)
|
||||
if _SECRET_CONTEXT_RE.search(window):
|
||||
count += 1
|
||||
return mask(match.group(0), '[HEX_KEY MASQUÉ]')
|
||||
return match.group(0)
|
||||
|
||||
return _HEX_RE.sub(_replace, text), count
|
||||
|
||||
|
||||
def _redact(text: str, mask) -> tuple[str, int]:
|
||||
"""Apply every pattern; ``mask(original, label) -> str`` builds the
|
||||
replacement (plain label, or placeholder for the interactive mode)."""
|
||||
count = 0
|
||||
result = text
|
||||
for pattern, replacement, secret_group in _PATTERNS:
|
||||
|
||||
def _sub(match: re.Match, replacement=replacement, secret_group=secret_group) -> str:
|
||||
if callable(replacement):
|
||||
label = replacement(match)
|
||||
else:
|
||||
label = match.expand(str(replacement))
|
||||
return mask(match.group(secret_group), label)
|
||||
|
||||
new_result, n = pattern.subn(_sub, result)
|
||||
count += n
|
||||
result = new_result
|
||||
result, hex_count = _redact_bare_hex_secrets(result, mask)
|
||||
return result, count + hex_count
|
||||
|
||||
|
||||
def _plain_mask(original: str, label: str) -> str:
|
||||
"""Plain masking: only the visible label survives."""
|
||||
return label
|
||||
|
||||
|
||||
def redact(text: str) -> tuple:
|
||||
"""Redact sensitive patterns from text.
|
||||
@@ -57,20 +188,84 @@ def redact(text: str) -> tuple:
|
||||
Returns:
|
||||
(redacted_text, redaction_count) tuple.
|
||||
"""
|
||||
count = 0
|
||||
result = text
|
||||
for pattern, replacement in _PATTERNS:
|
||||
if callable(replacement):
|
||||
new_result, n = pattern.subn(replacement, result)
|
||||
else:
|
||||
new_result, n = pattern.subn(str(replacement), result)
|
||||
count += n
|
||||
result = new_result
|
||||
result, count = _redact(text, _plain_mask)
|
||||
if count > 0:
|
||||
logger.info(f"Redacted {count} secret(s) from content")
|
||||
return result, count
|
||||
|
||||
|
||||
def redact_with_placeholders(text: str, file_path: str = "") -> tuple[str, list[tuple[str, str]]]:
|
||||
"""Redact *text*, replacing every mask with an opaque placeholder.
|
||||
|
||||
Used by the markdown rendering pipeline: placeholders survive the
|
||||
markdown → HTML conversion (fenced code blocks included, where a
|
||||
literal ``<span>`` would be shown as text), then
|
||||
:func:`restore_masks` turns them back into labels or clickable badges.
|
||||
|
||||
Args:
|
||||
text: The raw text content to scan.
|
||||
file_path: Optional file path for logging context.
|
||||
|
||||
Returns:
|
||||
(text_with_placeholders, entries) where *entries* is the list of
|
||||
``(label, secret)`` tuples referenced by the placeholders, in
|
||||
order of appearance.
|
||||
"""
|
||||
entries: list[tuple[str, str]] = []
|
||||
|
||||
def mask(original: str, label: str) -> str:
|
||||
entries.append((label, original))
|
||||
return f"{_PLACEHOLDER_OPEN}{len(entries) - 1}{_PLACEHOLDER_CLOSE}"
|
||||
|
||||
result, count = _redact(text, mask)
|
||||
if count > 0:
|
||||
logger.warning(f"Redacted {count} potential secret(s) from {file_path or '<unknown>'}")
|
||||
return result, entries
|
||||
|
||||
|
||||
def restore_masks(
|
||||
text: str,
|
||||
entries: list[tuple[str, str]],
|
||||
*,
|
||||
click_to_copy: bool = False,
|
||||
) -> str:
|
||||
"""Turn placeholders produced by :func:`redact_with_placeholders` back
|
||||
into visible masks.
|
||||
|
||||
Args:
|
||||
text: Rendered HTML still containing placeholders.
|
||||
entries: The ``(label, secret)`` list returned alongside.
|
||||
click_to_copy: When True (authenticated app preview), each mask
|
||||
becomes ``<span class="secret-mask" data-secret="…">label</span>``
|
||||
so a click copies the real value. When False (public shares,
|
||||
PDF exports), only the plain label is restored — the secret
|
||||
never reaches the page.
|
||||
|
||||
Returns:
|
||||
The text with every placeholder replaced.
|
||||
"""
|
||||
if not entries:
|
||||
return text
|
||||
|
||||
def _sub(match: re.Match) -> str:
|
||||
idx = int(match.group(1))
|
||||
if idx >= len(entries):
|
||||
return ""
|
||||
label, original = entries[idx]
|
||||
label_esc = _html.escape(str(label), quote=False)
|
||||
if not click_to_copy:
|
||||
return label_esc
|
||||
return (
|
||||
'<span class="secret-mask" data-secret="'
|
||||
+ _html.escape(str(original), quote=True)
|
||||
+ '">'
|
||||
+ label_esc
|
||||
+ "</span>"
|
||||
)
|
||||
|
||||
return _PLACEHOLDER_RE.sub(_sub, text)
|
||||
|
||||
|
||||
def redact_file_content(content: str, file_path: str = "") -> str:
|
||||
"""Redact a file's content for preview rendering.
|
||||
|
||||
|
||||
@@ -0,0 +1,608 @@
|
||||
"""ObsiGate — Semantic search: embeddings, vector store and hybrid retrieval.
|
||||
|
||||
This module adds a *semantic* layer on top of the existing TF-IDF search. Each
|
||||
document is split into overlapping chunks, each chunk is converted into a dense
|
||||
vector, and queries are matched by cosine similarity. Results are combined with
|
||||
the lexical ranking through Reciprocal Rank Fusion (RRF).
|
||||
|
||||
Design goals
|
||||
------------
|
||||
* **Zero mandatory dependency.** ``sentence-transformers`` (local model),
|
||||
``numpy`` and ``faiss`` are *optional*. They are imported lazily and, when
|
||||
missing, the module falls back to a deterministic pure-Python hashing embedder
|
||||
and a pure-Python cosine store. The feature therefore degrades gracefully and
|
||||
the default CI (which only installs ``backend/requirements.txt``) keeps working.
|
||||
* **Plug-in providers.** Embeddings can come from the local
|
||||
``all-MiniLM-L6-v2`` model, from an OpenAI-compatible ``/embeddings`` endpoint
|
||||
(configured via env vars), or from the deterministic fallback.
|
||||
* **Incremental.** The index is updated document-by-document from the indexer
|
||||
change hook (file watcher + API mutations), never rebuilt on each search.
|
||||
|
||||
Optional extras are listed in ``backend/requirements-semantic.txt``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import Counter
|
||||
from itertools import pairwise
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.semantic")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
EMBEDDING_DIM = 384 # all-MiniLM-L6-v2 output dimension
|
||||
CHUNK_TOKENS = 512 # target chunk size (whitespace tokens)
|
||||
CHUNK_OVERLAP_TOKENS = 64 # overlap between consecutive chunks
|
||||
DEFAULT_TOP_K = 200 # max documents returned by a semantic query
|
||||
RRF_K = 60 # Reciprocal Rank Fusion smoothing constant
|
||||
MAX_QUERY_CHARS = 2000 # guard against pathological queries
|
||||
|
||||
_WORD_RE = re.compile(r"[\w]+", re.UNICODE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenization / chunking
|
||||
# ---------------------------------------------------------------------------
|
||||
def _simple_tokens(text: str) -> list[str]:
|
||||
"""Split *text* into lowercase word tokens (keeps accents)."""
|
||||
return _WORD_RE.findall(text.lower())
|
||||
|
||||
|
||||
def chunk_text(
|
||||
text: str,
|
||||
chunk_tokens: int = CHUNK_TOKENS,
|
||||
overlap: int = CHUNK_OVERLAP_TOKENS,
|
||||
) -> list[str]:
|
||||
"""Split *text* into overlapping windows of roughly *chunk_tokens* words.
|
||||
|
||||
Args:
|
||||
text: Raw document text.
|
||||
chunk_tokens: Target number of whitespace tokens per chunk.
|
||||
overlap: Number of tokens shared by two consecutive chunks.
|
||||
|
||||
Returns:
|
||||
A list of chunk strings. Empty input yields an empty list.
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
if chunk_tokens <= 0:
|
||||
chunk_tokens = CHUNK_TOKENS
|
||||
overlap = max(0, min(overlap, chunk_tokens - 1))
|
||||
|
||||
words = text.split()
|
||||
if len(words) <= chunk_tokens:
|
||||
return [" ".join(words)]
|
||||
|
||||
step = max(1, chunk_tokens - overlap)
|
||||
chunks: list[str] = []
|
||||
for start in range(0, len(words), step):
|
||||
window = words[start:start + chunk_tokens]
|
||||
if not window:
|
||||
break
|
||||
chunks.append(" ".join(window))
|
||||
if start + chunk_tokens >= len(words):
|
||||
break
|
||||
return chunks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding providers
|
||||
# ---------------------------------------------------------------------------
|
||||
class EmbeddingProvider(ABC):
|
||||
"""Base class for embedding backends."""
|
||||
|
||||
name: str = "base"
|
||||
|
||||
def __init__(self, dimension: int = EMBEDDING_DIM) -> None:
|
||||
self.dimension = dimension
|
||||
|
||||
@abstractmethod
|
||||
def encode(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Return one L2-normalized vector per input text."""
|
||||
|
||||
def encode_one(self, text: str) -> list[float]:
|
||||
"""Convenience wrapper returning the vector for a single text."""
|
||||
vectors = self.encode([text])
|
||||
return vectors[0] if vectors else [0.0] * self.dimension
|
||||
|
||||
|
||||
class HashEmbeddingProvider(EmbeddingProvider):
|
||||
"""Deterministic, dependency-free hashing embedder.
|
||||
|
||||
This is a *lexical* fallback: it hashes word unigrams, word bigrams and
|
||||
character trigrams into fixed-size signed buckets (the "hashing trick"),
|
||||
then L2-normalizes the result. It captures shared vocabulary and
|
||||
morphological variants (``backup``/``backups``), so it already improves
|
||||
recall over exact TF-IDF matching, but it does not understand synonyms the
|
||||
way a real transformer model does.
|
||||
"""
|
||||
|
||||
name = "hash"
|
||||
|
||||
def _add_feature(self, vec: list[float], key: str, weight: float) -> None:
|
||||
digest = hashlib.blake2b(key.encode("utf-8"), digest_size=8).digest()
|
||||
h = int.from_bytes(digest, "big")
|
||||
idx = h % self.dimension
|
||||
sign = 1.0 if (h >> 63) & 1 else -1.0
|
||||
vec[idx] += sign * weight
|
||||
|
||||
def _encode_one(self, text: str) -> list[float]:
|
||||
vec = [0.0] * self.dimension
|
||||
tokens = _simple_tokens(text)
|
||||
if not tokens:
|
||||
return vec
|
||||
|
||||
tf = Counter(tokens)
|
||||
for token, count in tf.items():
|
||||
weight = 1.0 + math.log(count)
|
||||
self._add_feature(vec, "w:" + token, weight)
|
||||
for gram in _char_ngrams(token, 3):
|
||||
self._add_feature(vec, "g:" + gram, weight * 0.5)
|
||||
|
||||
for first, second in pairwise(tokens):
|
||||
self._add_feature(vec, "b:" + first + "_" + second, 0.5)
|
||||
|
||||
norm = math.sqrt(sum(v * v for v in vec))
|
||||
if norm > 0.0:
|
||||
vec = [v / norm for v in vec]
|
||||
return vec
|
||||
|
||||
def encode(self, texts: list[str]) -> list[list[float]]:
|
||||
return [self._encode_one(t or "") for t in texts]
|
||||
|
||||
|
||||
def _char_ngrams(token: str, n: int) -> list[str]:
|
||||
"""Return padded character n-grams for *token* (bounded to avoid blow-up)."""
|
||||
if len(token) < n:
|
||||
return [token]
|
||||
if len(token) > 24:
|
||||
token = token[:24]
|
||||
return [token[i:i + n] for i in range(len(token) - n + 1)]
|
||||
|
||||
|
||||
class SentenceTransformerProvider(EmbeddingProvider):
|
||||
"""Local ``all-MiniLM-L6-v2`` embeddings via ``sentence-transformers``."""
|
||||
|
||||
name = "sentence-transformers"
|
||||
|
||||
def __init__(self, model_name: str = "all-MiniLM-L6-v2") -> None:
|
||||
super().__init__(EMBEDDING_DIM)
|
||||
self.model_name = model_name
|
||||
self._model: Any | None = None
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
try:
|
||||
import sentence_transformers # noqa: F401
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _get_model(self) -> Any:
|
||||
if self._model is None:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
self._model = SentenceTransformer(self.model_name)
|
||||
return self._model
|
||||
|
||||
def encode(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
model = self._get_model()
|
||||
vectors = model.encode(texts, normalize_embeddings=True)
|
||||
return [[float(x) for x in vec] for vec in vectors]
|
||||
|
||||
|
||||
class RemoteEmbeddingProvider(EmbeddingProvider):
|
||||
"""OpenAI-compatible ``/embeddings`` endpoint (API key based)."""
|
||||
|
||||
name = "remote"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str = "text-embedding-3-small",
|
||||
dimension: int = EMBEDDING_DIM,
|
||||
) -> None:
|
||||
super().__init__(dimension)
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
|
||||
def encode(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
import httpx
|
||||
|
||||
response = httpx.post(
|
||||
f"{self.base_url}/embeddings",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={"model": self.model, "input": texts},
|
||||
timeout=30.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
data = sorted(payload.get("data", []), key=lambda item: item.get("index", 0))
|
||||
return [self._normalize([float(x) for x in item["embedding"]]) for item in data]
|
||||
|
||||
@staticmethod
|
||||
def _normalize(vec: list[float]) -> list[float]:
|
||||
norm = math.sqrt(sum(v * v for v in vec))
|
||||
if norm > 0.0:
|
||||
return [v / norm for v in vec]
|
||||
return vec
|
||||
|
||||
|
||||
_provider: EmbeddingProvider | None = None
|
||||
_provider_lock = threading.Lock()
|
||||
|
||||
|
||||
def _build_provider() -> EmbeddingProvider:
|
||||
"""Select the best available provider (respecting ``OBSIGATE_EMBEDDING_PROVIDER``)."""
|
||||
requested = os.getenv("OBSIGATE_EMBEDDING_PROVIDER", "auto").strip().lower()
|
||||
|
||||
if requested in ("auto", "local", "sentence-transformers") and SentenceTransformerProvider.is_available():
|
||||
model = os.getenv("OBSIGATE_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
|
||||
return SentenceTransformerProvider(model)
|
||||
|
||||
remote_key = os.getenv("OBSIGATE_EMBEDDING_API_KEY", "")
|
||||
remote_url = os.getenv("OBSIGATE_EMBEDDING_BASE_URL", "")
|
||||
if requested in ("auto", "remote") and remote_key and remote_url:
|
||||
model = os.getenv("OBSIGATE_EMBEDDING_MODEL", "text-embedding-3-small")
|
||||
dim = int(os.getenv("OBSIGATE_EMBEDDING_DIM", str(EMBEDDING_DIM)))
|
||||
return RemoteEmbeddingProvider(remote_url, remote_key, model, dim)
|
||||
|
||||
if requested == "remote":
|
||||
logger.warning(
|
||||
"OBSIGATE_EMBEDDING_PROVIDER=remote but OBSIGATE_EMBEDDING_API_KEY/BASE_URL missing; using hash fallback"
|
||||
)
|
||||
return HashEmbeddingProvider()
|
||||
|
||||
|
||||
def get_embedding_provider() -> EmbeddingProvider:
|
||||
"""Return the cached embedding provider (built on first use)."""
|
||||
global _provider
|
||||
with _provider_lock:
|
||||
if _provider is None:
|
||||
_provider = _build_provider()
|
||||
logger.info("Semantic embedding provider: %s (dim=%d)", _provider.name, _provider.dimension)
|
||||
return _provider
|
||||
|
||||
|
||||
def reset_embedding_provider() -> None:
|
||||
"""Forget the cached provider (used by tests and config reloads)."""
|
||||
global _provider
|
||||
with _provider_lock:
|
||||
_provider = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vector store
|
||||
# ---------------------------------------------------------------------------
|
||||
class VectorStore:
|
||||
"""In-memory vector store with optional numpy / faiss acceleration.
|
||||
|
||||
Vectors are always kept as Python lists (source of truth). A numpy matrix
|
||||
and/or a faiss ``IndexFlatIP`` are built lazily and invalidated on mutation.
|
||||
All vectors are expected to be L2-normalized, so the inner product equals
|
||||
the cosine similarity.
|
||||
"""
|
||||
|
||||
def __init__(self, dimension: int = EMBEDDING_DIM) -> None:
|
||||
self.dimension = dimension
|
||||
self._keys: list[str] = []
|
||||
self._chunks: list[str] = []
|
||||
self._vectors: list[list[float]] = []
|
||||
self._dirty = True
|
||||
self._numpy: Any | None = None
|
||||
self._numpy_checked = False
|
||||
self._matrix: Any | None = None
|
||||
self._faiss: Any | None = None
|
||||
self._faiss_checked = False
|
||||
self._faiss_index: Any | None = None
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._vectors)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._keys = []
|
||||
self._chunks = []
|
||||
self._vectors = []
|
||||
self._dirty = True
|
||||
|
||||
def add(self, key: str, chunk: str, vector: list[float]) -> None:
|
||||
self._keys.append(key)
|
||||
self._chunks.append(chunk)
|
||||
self._vectors.append(vector)
|
||||
self._dirty = True
|
||||
|
||||
def remove_document(self, key: str) -> None:
|
||||
"""Remove every chunk belonging to *key*."""
|
||||
kept = [(k, c, v) for k, c, v in zip(self._keys, self._chunks, self._vectors) if k != key]
|
||||
if len(kept) == len(self._vectors):
|
||||
return
|
||||
self._keys = [k for k, _, _ in kept]
|
||||
self._chunks = [c for _, c, _ in kept]
|
||||
self._vectors = [v for _, _, v in kept]
|
||||
self._dirty = True
|
||||
|
||||
# -- optional accelerators -------------------------------------------------
|
||||
def _get_numpy(self) -> Any | None:
|
||||
if not self._numpy_checked:
|
||||
self._numpy_checked = True
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
self._numpy = np
|
||||
except Exception:
|
||||
self._numpy = None
|
||||
return self._numpy
|
||||
|
||||
def _get_faiss(self) -> Any | None:
|
||||
if not self._faiss_checked:
|
||||
self._faiss_checked = True
|
||||
try:
|
||||
import faiss
|
||||
|
||||
self._faiss = faiss
|
||||
except Exception:
|
||||
self._faiss = None
|
||||
return self._faiss
|
||||
|
||||
def _rebuild_accelerators(self) -> None:
|
||||
self._dirty = False
|
||||
np = self._get_numpy()
|
||||
if np is None or not self._vectors:
|
||||
self._matrix = None
|
||||
self._faiss_index = None
|
||||
return
|
||||
self._matrix = np.asarray(self._vectors, dtype="float32")
|
||||
faiss = self._get_faiss()
|
||||
if faiss is not None:
|
||||
index = faiss.IndexFlatIP(self.dimension)
|
||||
index.add(self._matrix)
|
||||
self._faiss_index = index
|
||||
else:
|
||||
self._faiss_index = None
|
||||
|
||||
def search(self, query_vector: list[float], top_k: int = DEFAULT_TOP_K) -> list[tuple[str, float]]:
|
||||
"""Return ``(doc_key, cosine_similarity)`` pairs sorted by similarity."""
|
||||
if not self._vectors:
|
||||
return []
|
||||
top_k = max(1, min(top_k, len(self._vectors)))
|
||||
if self._dirty:
|
||||
self._rebuild_accelerators()
|
||||
|
||||
np = self._get_numpy()
|
||||
if np is not None and self._faiss_index is not None and self._matrix is not None:
|
||||
query = np.asarray([query_vector], dtype="float32")
|
||||
scores, indices = self._faiss_index.search(query, top_k)
|
||||
return [
|
||||
(self._keys[int(idx)], float(score))
|
||||
for score, idx in zip(scores[0], indices[0])
|
||||
if idx >= 0
|
||||
]
|
||||
|
||||
if np is not None and self._matrix is not None:
|
||||
query = np.asarray(query_vector, dtype="float32")
|
||||
scores = self._matrix @ query
|
||||
order = np.argsort(scores)[::-1][:top_k]
|
||||
return [(self._keys[int(i)], float(scores[int(i)])) for i in order]
|
||||
|
||||
scored = [(self._keys[i], _dot(self._vectors[i], query_vector)) for i in range(len(self._vectors))]
|
||||
scored.sort(key=lambda item: item[1], reverse=True)
|
||||
return scored[:top_k]
|
||||
|
||||
def chunk_of(self, index: int) -> str:
|
||||
"""Return the stored chunk text at *index* (used by diagnostics/tests)."""
|
||||
return self._chunks[index]
|
||||
|
||||
|
||||
def _dot(a: list[float], b: list[float]) -> float:
|
||||
"""Dot product for two equal-length vectors."""
|
||||
return sum(x * y for x, y in zip(a, b))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reciprocal Rank Fusion
|
||||
# ---------------------------------------------------------------------------
|
||||
def rrf_fuse(rankings: list[list[str]], k: int = RRF_K) -> dict[str, float]:
|
||||
"""Fuse several ranked key lists into a single score map.
|
||||
|
||||
``score(key) = Σ_rankings 1 / (k + rank(key))`` where ``rank`` is
|
||||
1-based. Documents ranked highly by several methods rise to the top.
|
||||
|
||||
Args:
|
||||
rankings: Ordered lists of document keys (best first).
|
||||
k: RRF smoothing constant.
|
||||
|
||||
Returns:
|
||||
Mapping ``doc_key -> fused score`` (insertion order is unspecified).
|
||||
"""
|
||||
scores: dict[str, float] = {}
|
||||
for ranking in rankings:
|
||||
seen: set[str] = set()
|
||||
for rank, key in enumerate(ranking):
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
scores[key] = scores.get(key, 0.0) + 1.0 / (k + rank + 1)
|
||||
return scores
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Semantic index
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticIndex:
|
||||
"""Holds document chunk embeddings and answers similarity queries."""
|
||||
|
||||
def __init__(self, provider: EmbeddingProvider | None = None) -> None:
|
||||
self.provider = provider
|
||||
self.store = VectorStore(provider.dimension if provider else EMBEDDING_DIM)
|
||||
self.doc_keys: set[str] = set()
|
||||
self._ready = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
"""Return True once a full rebuild has completed."""
|
||||
return self._ready
|
||||
|
||||
def _ensure_provider(self) -> EmbeddingProvider:
|
||||
if self.provider is None:
|
||||
self.provider = get_embedding_provider()
|
||||
self.store = VectorStore(self.provider.dimension)
|
||||
return self.provider
|
||||
|
||||
@staticmethod
|
||||
def _document_text(file_info: dict[str, Any]) -> str:
|
||||
title = file_info.get("title", "") or ""
|
||||
content = file_info.get("content", "") or ""
|
||||
return (title + "\n\n" + content).strip()
|
||||
|
||||
def _embed_document(self, doc_key: str, file_info: dict[str, Any]) -> None:
|
||||
text = self._document_text(file_info)
|
||||
if not text:
|
||||
return
|
||||
provider = self._ensure_provider()
|
||||
chunks = chunk_text(text)
|
||||
if not chunks:
|
||||
return
|
||||
vectors = provider.encode(chunks)
|
||||
for chunk, vector in zip(chunks, vectors):
|
||||
self.store.add(doc_key, chunk, vector)
|
||||
self.doc_keys.add(doc_key)
|
||||
|
||||
def rebuild(self) -> None:
|
||||
"""Rebuild the whole index from the global in-memory index."""
|
||||
from backend.indexer import index
|
||||
|
||||
provider = self._ensure_provider()
|
||||
with self._lock:
|
||||
self.store = VectorStore(provider.dimension)
|
||||
self.doc_keys = set()
|
||||
for vault_name, vault_data in index.items():
|
||||
for file_info in vault_data.get("files", []):
|
||||
doc_key = f"{vault_name}::{file_info.get('path', '')}"
|
||||
try:
|
||||
self._embed_document(doc_key, file_info)
|
||||
except Exception as exc:
|
||||
logger.warning("Semantic embedding failed for %s: %s", doc_key, exc)
|
||||
self._ready = True
|
||||
logger.info(
|
||||
"Semantic index built: %d documents, %d chunks (provider=%s)",
|
||||
len(self.doc_keys),
|
||||
len(self.store),
|
||||
provider.name,
|
||||
)
|
||||
|
||||
def add_document(self, vault_name: str, path: str, file_info: dict[str, Any]) -> None:
|
||||
"""Add or refresh a single document (no-op until the index is ready)."""
|
||||
if not self._ready or not file_info:
|
||||
return
|
||||
doc_key = f"{vault_name}::{path}"
|
||||
with self._lock:
|
||||
self.store.remove_document(doc_key)
|
||||
self.doc_keys.discard(doc_key)
|
||||
try:
|
||||
self._embed_document(doc_key, file_info)
|
||||
except Exception as exc:
|
||||
logger.warning("Semantic embedding failed for %s: %s", doc_key, exc)
|
||||
|
||||
def remove_document(self, vault_name: str, path: str) -> None:
|
||||
"""Remove a single document (no-op until the index is ready)."""
|
||||
if not self._ready:
|
||||
return
|
||||
doc_key = f"{vault_name}::{path}"
|
||||
with self._lock:
|
||||
self.store.remove_document(doc_key)
|
||||
self.doc_keys.discard(doc_key)
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
vault_filter: str = "all",
|
||||
top_k: int = DEFAULT_TOP_K,
|
||||
) -> list[tuple[str, float]]:
|
||||
"""Return ``(doc_key, best_chunk_similarity)`` pairs, best first."""
|
||||
if not self._ready or not query or not query.strip():
|
||||
return []
|
||||
provider = self._ensure_provider()
|
||||
query_vector = provider.encode_one(query[:MAX_QUERY_CHARS])
|
||||
hits = self.store.search(query_vector, top_k=max(top_k * 4, top_k))
|
||||
best: dict[str, float] = {}
|
||||
for doc_key, score in hits:
|
||||
if vault_filter != "all" and not doc_key.startswith(vault_filter + "::"):
|
||||
continue
|
||||
if doc_key not in best or score > best[doc_key]:
|
||||
best[doc_key] = score
|
||||
ordered = sorted(best.items(), key=lambda item: item[1], reverse=True)
|
||||
return ordered[:top_k]
|
||||
|
||||
|
||||
_semantic_index: SemanticIndex | None = None
|
||||
_index_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_semantic_index() -> SemanticIndex:
|
||||
"""Return the process-wide semantic index (created on first access)."""
|
||||
global _semantic_index
|
||||
with _index_lock:
|
||||
if _semantic_index is None:
|
||||
_semantic_index = SemanticIndex()
|
||||
return _semantic_index
|
||||
|
||||
|
||||
def reset_semantic_index() -> None:
|
||||
"""Drop the singleton index (tests)."""
|
||||
global _semantic_index
|
||||
with _index_lock:
|
||||
_semantic_index = None
|
||||
|
||||
|
||||
def init_semantic_index() -> None:
|
||||
"""Force a full semantic index build. Called after ``build_index`` on startup."""
|
||||
from backend.indexer import index
|
||||
|
||||
if any(vdata.get("files") for vdata in index.values()):
|
||||
get_semantic_index().rebuild()
|
||||
|
||||
|
||||
def on_index_change(action: str, vault_name: str, path: str, file_info: dict[str, Any]) -> None:
|
||||
"""Incremental hook registered with the indexer change notifier."""
|
||||
index_obj = get_semantic_index()
|
||||
if action == "add" and file_info:
|
||||
index_obj.add_document(vault_name, path, file_info)
|
||||
elif action == "remove":
|
||||
index_obj.remove_document(vault_name, path)
|
||||
|
||||
|
||||
def semantic_search_docs(
|
||||
query: str,
|
||||
vault_filter: str = "all",
|
||||
top_k: int = DEFAULT_TOP_K,
|
||||
) -> list[tuple[str, float]]:
|
||||
"""Convenience wrapper around :meth:`SemanticIndex.search`."""
|
||||
return get_semantic_index().search(query, vault_filter=vault_filter, top_k=top_k)
|
||||
|
||||
|
||||
def semantic_status() -> dict[str, Any]:
|
||||
"""Return provider/index diagnostics for the API and the UI."""
|
||||
index_obj = get_semantic_index()
|
||||
provider = index_obj.provider or get_embedding_provider()
|
||||
return {
|
||||
"available": index_obj.is_ready(),
|
||||
"provider": provider.name,
|
||||
"dimension": provider.dimension,
|
||||
"documents": len(index_obj.doc_keys),
|
||||
"chunks": len(index_obj.store),
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Backup inspection services shared by REST routes and the AI tool layer.
|
||||
|
||||
Single source of truth for locating, listing and diffing the timestamped
|
||||
backups created before each file mutation. The write-side (creating a backup)
|
||||
stays in the route layer; only the read/inspection logic lives here so both
|
||||
``/api/file/{vault}/backups``, ``/api/file/{vault}/diff`` and the tools
|
||||
``list_backups`` / ``diff_backup`` behave identically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
logger = logging.getLogger("obsigate.services.backups")
|
||||
|
||||
# Default number of backups kept per file (matches ``max_backups_per_file``).
|
||||
DEFAULT_MAX_BACKUPS = 10
|
||||
|
||||
|
||||
def _default_max_backups() -> int:
|
||||
"""Read ``max_backups_per_file`` from app config (lazy, best-effort)."""
|
||||
try:
|
||||
from backend.routers.config import _load_config # ROADMAP #85 T7 — déménagé depuis backend.main
|
||||
|
||||
return int(_load_config().get("max_backups_per_file", DEFAULT_MAX_BACKUPS))
|
||||
except Exception: # pragma: no cover - config unavailable
|
||||
return DEFAULT_MAX_BACKUPS
|
||||
|
||||
|
||||
def get_backup_dir(vault_name: str, relative_path: str) -> Path:
|
||||
"""Return the directory where backups for a specific file are stored.
|
||||
|
||||
Resolves relative backup paths against the SPECIFIC vault's directory,
|
||||
matching the backup-creation logic exactly.
|
||||
"""
|
||||
from backend.indexer import get_vault_data
|
||||
|
||||
backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup"))
|
||||
if not backup_root.is_absolute():
|
||||
vault_data = get_vault_data(vault_name)
|
||||
if vault_data:
|
||||
backup_root = Path(vault_data["path"]) / backup_root
|
||||
return backup_root / vault_name / Path(relative_path).parent
|
||||
|
||||
|
||||
def create_backup(
|
||||
file_path: Path,
|
||||
vault_name: str,
|
||||
relative_path: str,
|
||||
*,
|
||||
max_backups: int | None = None,
|
||||
) -> Path | None:
|
||||
"""Create a timestamped backup of a file before it is modified.
|
||||
|
||||
Backups are stored as ``{backup_root}/{vault}/{relative_path}.{ts}.bak``.
|
||||
Missing or unreadable files are skipped silently (returns ``None``) so a
|
||||
backup failure never blocks the caller's mutation.
|
||||
|
||||
Args:
|
||||
file_path: Absolute path of the file to back up.
|
||||
vault_name: Name of the vault the file belongs to.
|
||||
relative_path: Vault-relative path (used to mirror the tree).
|
||||
max_backups: Number of backups to keep per file. ``None`` reads
|
||||
``max_backups_per_file`` from the app config.
|
||||
|
||||
Returns:
|
||||
The created backup path, or ``None`` when skipped.
|
||||
"""
|
||||
try:
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
logger.debug(f"Backup skipped: file not found {file_path}")
|
||||
return None
|
||||
|
||||
backup_dir = get_backup_dir(vault_name, relative_path)
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
backup_path = backup_dir / f"{file_path.name}.{int(time.time())}.bak"
|
||||
shutil.copy2(file_path, backup_path)
|
||||
logger.info(f"Backup created: {relative_path} -> {backup_path}")
|
||||
|
||||
keep = max_backups if max_backups is not None else _default_max_backups()
|
||||
all_backups = sorted(
|
||||
[
|
||||
f
|
||||
for f in backup_dir.iterdir()
|
||||
if f.is_file() and f.name.startswith(file_path.name + ".") and f.name.endswith(".bak")
|
||||
],
|
||||
key=lambda f: f.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
for old in all_backups[keep:]:
|
||||
old.unlink()
|
||||
logger.debug(f"Auto-cleanup: removed old backup {old.name}")
|
||||
return backup_path
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to backup {relative_path} (vault={vault_name}): {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def list_backup_files(vault_name: str, relative_path: str) -> list[dict[str, Any]]:
|
||||
"""List all backup files for a vault file, sorted newest first.
|
||||
|
||||
Backup filename format: ``{original_filename}.{timestamp}.bak``.
|
||||
|
||||
Returns a list of dicts with ``timestamp``, ``datetime``, ``size`` and
|
||||
``filename``. Missing or unreadable backup directories yield an empty list.
|
||||
"""
|
||||
backup_dir = get_backup_dir(vault_name, relative_path)
|
||||
if not backup_dir.exists():
|
||||
return []
|
||||
|
||||
original_name = Path(relative_path).name
|
||||
prefix = original_name + "."
|
||||
backups: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
dir_entries = list(backup_dir.iterdir())
|
||||
except PermissionError:
|
||||
logger.warning(f"Permission denied reading backup dir: {backup_dir}")
|
||||
return []
|
||||
except OSError as e:
|
||||
logger.error(f"Error reading backup dir {backup_dir}: {e}")
|
||||
return []
|
||||
|
||||
for f in dir_entries:
|
||||
try:
|
||||
if not f.is_file():
|
||||
continue
|
||||
name = f.name
|
||||
if not name.startswith(prefix) or not name.endswith(".bak"):
|
||||
continue
|
||||
ts_part = name[len(prefix):-len(".bak")]
|
||||
try:
|
||||
ts = int(ts_part)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
try:
|
||||
dt = datetime.fromtimestamp(ts, tz=UTC).isoformat()
|
||||
except (OSError, OverflowError, ValueError) as ts_err:
|
||||
logger.warning(f"Skipping backup with invalid timestamp {ts}: {ts_err}")
|
||||
continue
|
||||
st = f.stat()
|
||||
backups.append({
|
||||
"timestamp": ts,
|
||||
"datetime": dt,
|
||||
"size": st.st_size,
|
||||
"filename": name,
|
||||
})
|
||||
except Exception as entry_err:
|
||||
logger.warning(f"Skipping unreadable backup entry {f}: {entry_err}")
|
||||
continue
|
||||
|
||||
backups.sort(key=lambda b: b["timestamp"], reverse=True)
|
||||
return backups
|
||||
|
||||
|
||||
def diff_backup(
|
||||
vault_name: str,
|
||||
path: str,
|
||||
version: int,
|
||||
compare_with: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a unified diff between a backup version and another version or the current file.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative path of the file within the vault.
|
||||
version: Timestamp of the backup used as the old/left side.
|
||||
compare_with: Optional timestamp of another backup as the new/right
|
||||
side. If omitted, the current file on disk is used.
|
||||
|
||||
Returns:
|
||||
Dict with ``vault``, ``path``, ``version``, ``compare_with``, ``diff``,
|
||||
``left_content`` and ``right_content``.
|
||||
|
||||
Raises:
|
||||
ServiceError: ``not_found`` (404) when the file or a backup is missing,
|
||||
``read_error`` (500) when a file cannot be read.
|
||||
"""
|
||||
root = get_vault_root(vault_name)
|
||||
file_path = resolve_safe_path(root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise ServiceError(
|
||||
f"File not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
original_name = Path(path).name
|
||||
|
||||
def _read_backup(ts: int) -> tuple[str, str]:
|
||||
backup_dir = get_backup_dir(vault_name, path)
|
||||
backup_path = backup_dir / f"{original_name}.{ts}.bak"
|
||||
if not backup_path.exists():
|
||||
raise ServiceError(
|
||||
f"Backup version {ts} not found for {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path, "version": ts},
|
||||
)
|
||||
try:
|
||||
content = backup_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
raise ServiceError(
|
||||
f"Failed to read backup {ts}: {e}",
|
||||
code="read_error",
|
||||
status=500,
|
||||
details={"vault": vault_name, "path": path, "version": ts},
|
||||
) from e
|
||||
dt = datetime.fromtimestamp(ts, tz=UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
return content, f"{path}@{dt}"
|
||||
|
||||
try:
|
||||
left_content, left_label = _read_backup(version)
|
||||
|
||||
if compare_with is not None:
|
||||
right_content, right_label = _read_backup(compare_with)
|
||||
else:
|
||||
try:
|
||||
right_content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
raise ServiceError(
|
||||
f"Failed to read current file: {e}",
|
||||
code="read_error",
|
||||
status=500,
|
||||
details={"vault": vault_name, "path": path},
|
||||
) from e
|
||||
right_label = f"{path} (current)"
|
||||
|
||||
left_lines = left_content.splitlines(keepends=True)
|
||||
right_lines = right_content.splitlines(keepends=True)
|
||||
|
||||
diff_lines = list(difflib.unified_diff(
|
||||
left_lines, right_lines,
|
||||
fromfile=left_label, tofile=right_label,
|
||||
))
|
||||
|
||||
return {
|
||||
"vault": vault_name,
|
||||
"path": path,
|
||||
"version": version,
|
||||
"compare_with": compare_with,
|
||||
"diff": "".join(diff_lines),
|
||||
"left_content": left_content,
|
||||
"right_content": right_content,
|
||||
}
|
||||
except ServiceError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating diff for {vault_name}/{path}: {type(e).__name__}: {e}", exc_info=True)
|
||||
raise ServiceError(f"Erreur lors de la génération du diff: {e!s}", code="read_error", status=500) from e
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Duplicate detection & merge services (#166).
|
||||
|
||||
Single source of truth consumed by the REST routes
|
||||
(``/api/duplicates``) and the AI tool layer (``find_duplicates``,
|
||||
``merge_duplicate_notes``).
|
||||
|
||||
Method is deterministic stdlib-only: frontmatter stripped, token-set
|
||||
Jaccard blended with a title similarity. No embedding dependency —
|
||||
the semantic index (#70) stays an optional refinement, not a requirement.
|
||||
|
||||
Fusion never runs without an explicit confirmation: the tool layer
|
||||
registers the merge as ``DANGEROUS`` (two-step propose/apply) and this
|
||||
service takes an automatic backup before any destructive write.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.services.backups import create_backup
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
logger = logging.getLogger("obsigate.services.duplicates")
|
||||
|
||||
MAX_FILES_SCANNED = 500
|
||||
MAX_FILE_BYTES = 200_000
|
||||
MAX_CONTENT_CHARS = 50_000
|
||||
|
||||
_WORD_RE = re.compile(r"[\w]+", re.UNICODE)
|
||||
_FRONTMATTER_RE = re.compile(r"\A---\s*\n.*?\n---\s*\n", re.DOTALL)
|
||||
|
||||
|
||||
def _strip_frontmatter(text: str) -> str:
|
||||
"""Remove a leading YAML frontmatter block, if present."""
|
||||
return _FRONTMATTER_RE.sub("", text, count=1)
|
||||
|
||||
|
||||
def _tokens(text: str) -> set[str]:
|
||||
"""Lowercase word tokens (keeps accents), stop-words free but tiny tokens dropped."""
|
||||
return {t for t in _WORD_RE.findall(text.lower()) if len(t) > 2}
|
||||
|
||||
|
||||
def similarity_score(a: str, b: str) -> float:
|
||||
"""Blend Jaccard (0.7) + title/first-line similarity (0.3) in [0, 1].
|
||||
|
||||
Pure function — unit-tested directly.
|
||||
"""
|
||||
ta, tb = _tokens(_strip_frontmatter(a)), _tokens(_strip_frontmatter(b))
|
||||
if not ta or not tb:
|
||||
return 0.0
|
||||
jaccard = len(ta & tb) / len(ta | tb)
|
||||
head_a = (a.strip().splitlines() or [""])[:1][0][:200].lower()
|
||||
head_b = (b.strip().splitlines() or [""])[:1][0][:200].lower()
|
||||
title_sim = SequenceMatcher(None, head_a, head_b).ratio() if head_a and head_b else 0.0
|
||||
return round(0.7 * jaccard + 0.3 * title_sim, 4)
|
||||
|
||||
|
||||
def _iter_markdown_files(root: Path, subdir: str = "") -> list[Path]:
|
||||
base = resolve_safe_path(root, subdir) if subdir else root.resolve()
|
||||
if not base.exists() or not base.is_dir():
|
||||
raise ServiceError(
|
||||
f"Directory not found: {subdir or '.'}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": subdir},
|
||||
)
|
||||
files = sorted(
|
||||
(p for p in base.rglob("*.md") if p.is_file() and not p.is_symlink()),
|
||||
key=lambda p: str(p),
|
||||
)
|
||||
return files[:MAX_FILES_SCANNED]
|
||||
|
||||
|
||||
def _read_capped(path: Path) -> str:
|
||||
try:
|
||||
if path.stat().st_size > MAX_FILE_BYTES:
|
||||
return ""
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
return text[:MAX_CONTENT_CHARS]
|
||||
|
||||
|
||||
def find_duplicate_pairs(
|
||||
vault: str,
|
||||
threshold: float = 0.75,
|
||||
limit: int = 50,
|
||||
subdir: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Return candidate duplicate pairs ordered by descending score.
|
||||
|
||||
Args:
|
||||
vault: Vault name.
|
||||
threshold: Minimum blended score in [0.3, 1.0].
|
||||
limit: Max pairs returned (1-200).
|
||||
subdir: Optional vault-relative directory scope.
|
||||
"""
|
||||
if not 0.3 <= threshold <= 1.0:
|
||||
raise ServiceError(
|
||||
"threshold must be between 0.3 and 1.0",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
limit = max(1, min(limit, 200))
|
||||
root = get_vault_root(vault)
|
||||
files = _iter_markdown_files(root, subdir)
|
||||
contents: dict[str, str] = {}
|
||||
token_sets: dict[str, set[str]] = {}
|
||||
for path in files:
|
||||
rel = str(path.relative_to(root)).replace("\\", "/")
|
||||
text = _read_capped(path)
|
||||
if not text.strip():
|
||||
continue
|
||||
contents[rel] = text
|
||||
token_sets[rel] = _tokens(_strip_frontmatter(text))
|
||||
|
||||
rels = sorted(contents)
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for i in range(len(rels)):
|
||||
for j in range(i + 1, len(rels)):
|
||||
a, b = rels[i], rels[j]
|
||||
ta, tb = token_sets[a], token_sets[b]
|
||||
if not ta or not tb:
|
||||
continue
|
||||
# Cheap pre-filter: Jaccard lower bound before the full score.
|
||||
inter = len(ta & tb)
|
||||
union = len(ta | tb)
|
||||
if union == 0 or inter / union < threshold * 0.6:
|
||||
continue
|
||||
score = similarity_score(contents[a], contents[b])
|
||||
if score >= threshold:
|
||||
pairs.append({"file_a": a, "file_b": b, "score": score})
|
||||
pairs.sort(key=lambda p: p["score"], reverse=True)
|
||||
return {
|
||||
"vault": vault,
|
||||
"threshold": threshold,
|
||||
"files_scanned": len(contents),
|
||||
"truncated": len(files) >= MAX_FILES_SCANNED,
|
||||
"pairs": pairs[:limit],
|
||||
}
|
||||
|
||||
|
||||
def merge_duplicates(
|
||||
vault: str,
|
||||
source_path: str,
|
||||
target_path: str,
|
||||
strategy: str = "append",
|
||||
) -> dict[str, Any]:
|
||||
"""Merge *source_path* into *target_path*, then delete the source.
|
||||
|
||||
Strategies:
|
||||
``append`` — source content appended after target (separator + origin
|
||||
marker), source deleted.
|
||||
``prefer_target`` — source deleted, target untouched (dedupe only).
|
||||
``prefer_source`` — target overwritten with source content, source deleted.
|
||||
|
||||
A backup of both files is taken first; the source deletion also goes
|
||||
through the backup-aware mutation service.
|
||||
"""
|
||||
from backend.services import mutations as _mutations
|
||||
|
||||
if strategy not in ("append", "prefer_target", "prefer_source"):
|
||||
raise ServiceError(
|
||||
f"Unknown strategy: {strategy}",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
if source_path == target_path:
|
||||
raise ServiceError(
|
||||
"source_path and target_path must differ",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
root = get_vault_root(vault)
|
||||
src = resolve_safe_path(root, source_path)
|
||||
dst = resolve_safe_path(root, target_path)
|
||||
if not src.is_file() or src.suffix.lower() != ".md":
|
||||
raise ServiceError(
|
||||
f"Source not found: {source_path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": source_path},
|
||||
)
|
||||
if not dst.is_file() or dst.suffix.lower() != ".md":
|
||||
raise ServiceError(
|
||||
f"Target not found: {target_path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": target_path},
|
||||
)
|
||||
# Backup préalable (jamais de fusion sans filet — critère #166).
|
||||
create_backup(src, vault, source_path)
|
||||
create_backup(dst, vault, target_path)
|
||||
|
||||
if strategy == "prefer_target":
|
||||
result = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "deleted": source_path, "delete": result}
|
||||
if strategy == "prefer_source":
|
||||
content = src.read_text(encoding="utf-8", errors="replace")
|
||||
result = _mutations.edit_file(vault, target_path, content)
|
||||
deleted = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|
||||
# append
|
||||
target_text = dst.read_text(encoding="utf-8", errors="replace")
|
||||
source_text = src.read_text(encoding="utf-8", errors="replace")
|
||||
merged = target_text.rstrip() + f"\n\n---\n\n_Fusionné depuis `{source_path}` (#166)_\n\n" + source_text.lstrip()
|
||||
result = _mutations.edit_file(vault, target_path, merged)
|
||||
deleted = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Domain errors shared by the reusable service layer.
|
||||
|
||||
Services are transport-agnostic: they raise :class:`ServiceError` carrying a
|
||||
stable ``code`` and an HTTP ``status`` hint. The REST layer maps it to an
|
||||
``HTTPException`` (via the global handler in ``backend.main``) while the tool
|
||||
layer maps it to a :class:`backend.tools.context.ToolError`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ServiceError(Exception):
|
||||
"""Base error for the shared business-logic services."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
code: str = "service_error",
|
||||
status: int = 400,
|
||||
details: dict[str, Any] | None = None,
|
||||
):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
self.status = status
|
||||
self.details = details or {}
|
||||
@@ -0,0 +1,104 @@
|
||||
"""File reading services shared by REST routes and the AI tool layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import chemin_exclu, get_vault_root
|
||||
|
||||
logger = logging.getLogger("obsigate.services.files")
|
||||
|
||||
|
||||
def _ensure_not_excluded(vault_name: str, path: str) -> None:
|
||||
"""Raise ``not_found`` (404) when *path* is excluded in this vault.
|
||||
|
||||
Un chemin exclu (DIR_N_IGNORED_DIRS / DIR_N_IGNORED_PATHS) est
|
||||
indiscernable d'un chemin inexistant : même erreur, aucun détail
|
||||
sur la règle qui l'a caché.
|
||||
"""
|
||||
if chemin_exclu(vault_name, path):
|
||||
raise ServiceError(
|
||||
f"File not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
|
||||
def read_raw_file(vault_name: str, path: str) -> dict[str, Any]:
|
||||
"""Return the raw text content of a vault file (no redaction)."""
|
||||
_ensure_not_excluded(vault_name, path)
|
||||
root = get_vault_root(vault_name)
|
||||
file_path = resolve_safe_path(root, path)
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise ServiceError(
|
||||
f"File not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
except PermissionError as e:
|
||||
logger.error(f"Permission denied reading raw file {path}: {e}")
|
||||
raise ServiceError(f"Permission denied: cannot read file {path}", code="permission_denied", status=403) from e
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
raw = file_path.read_bytes().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading binary raw file {path}: {e}")
|
||||
raise ServiceError(f"Cannot read file: {e!s}", code="read_error", status=500) from e
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error reading raw file {path}: {e}")
|
||||
raise ServiceError(f"Error reading file: {e!s}", code="read_error", status=500) from e
|
||||
|
||||
return {"vault": vault_name, "path": path, "raw": raw}
|
||||
|
||||
|
||||
def read_file_text(
|
||||
vault_name: str,
|
||||
path: str,
|
||||
*,
|
||||
redact: bool = True,
|
||||
max_bytes: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a vault file's text content, optionally redacted and size-capped.
|
||||
|
||||
Raises:
|
||||
ServiceError: ``not_found`` (404), ``file_too_large`` (413) or a read
|
||||
error (500).
|
||||
"""
|
||||
_ensure_not_excluded(vault_name, path)
|
||||
root = get_vault_root(vault_name)
|
||||
target = resolve_safe_path(root, path)
|
||||
|
||||
if not target.exists() or not target.is_file():
|
||||
raise ServiceError(
|
||||
f"File not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
size = target.stat().st_size
|
||||
if max_bytes is not None and size > max_bytes:
|
||||
raise ServiceError(
|
||||
f"File too large ({size} bytes > {max_bytes})",
|
||||
code="file_too_large",
|
||||
status=413,
|
||||
details={"vault": vault_name, "path": path, "size": size},
|
||||
)
|
||||
|
||||
content = target.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
if redact:
|
||||
from backend.secret_redactor import redact_file_content
|
||||
|
||||
content = redact_file_content(content, path)
|
||||
|
||||
return {"vault": vault_name, "path": path, "size": size, "content": content}
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Graph data services shared by the REST route and the AI tool layer.
|
||||
|
||||
Builds the nodes/edges structure consumed by the graph view. The route
|
||||
``/api/graph/{vault}`` and the tool ``get_graph`` both delegate here so the
|
||||
graph semantics (parent/child edges, wikilinks, tag filter) stay identical.
|
||||
Permission checks are the caller's responsibility.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.indexer import SUPPORTED_EXTENSIONS, index
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
_WIKILINK_PATTERN = re.compile(r"\[\[([^\]|#]+)(?:[|#][^\]]+)?\]\]")
|
||||
|
||||
|
||||
def get_graph(
|
||||
vault_name: str,
|
||||
path: str = "",
|
||||
depth: int = 1,
|
||||
scope: str = "directory",
|
||||
tag: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Return graph data (nodes and edges) for a vault or directory.
|
||||
|
||||
Args:
|
||||
vault_name: Name of the vault.
|
||||
path: Relative directory path to focus on (empty = root).
|
||||
depth: Expansion depth (0 = only direct children, 1-3 = deeper).
|
||||
scope: ``"directory"`` for a subtree, ``"full"`` for the whole vault.
|
||||
tag: Optional tag filter (only files with this tag appear).
|
||||
|
||||
Returns:
|
||||
Dict with ``vault``, ``path``, ``scope``, ``nodes`` and ``edges``.
|
||||
|
||||
Raises:
|
||||
ServiceError: ``not_found`` (404) when the vault or path is missing.
|
||||
"""
|
||||
from backend.vault_settings import get_vault_setting
|
||||
|
||||
vault_root = get_vault_root(vault_name)
|
||||
target = resolve_safe_path(vault_root, path) if path else vault_root.resolve()
|
||||
|
||||
if not target.exists():
|
||||
raise ServiceError(
|
||||
f"Path not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
nodes: list[dict[str, Any]] = []
|
||||
edges: list[dict[str, Any]] = []
|
||||
node_ids: set[str] = set()
|
||||
|
||||
def _add_node(name: str, ntype: str, npath: str, size: int = 0,
|
||||
tags: list[str] | None = None, incoming: int = 0, outgoing: int = 0) -> str:
|
||||
nid = f"{vault_name}:{npath}"
|
||||
if nid not in node_ids:
|
||||
node_ids.add(nid)
|
||||
nodes.append({
|
||||
"id": nid, "name": name, "type": ntype, "path": npath,
|
||||
"size": size, "tags": tags or [],
|
||||
"incoming_count": incoming, "outgoing_count": outgoing,
|
||||
})
|
||||
return nid
|
||||
|
||||
def _add_edge(source: str, target_id: str, relation: str) -> None:
|
||||
edges.append({"source": source, "target": target_id, "relation": relation})
|
||||
|
||||
settings = get_vault_setting(vault_name) or {}
|
||||
hide_hidden = settings.get("hideHiddenFiles", False)
|
||||
|
||||
# Build tag index from the in-memory index for fast lookups.
|
||||
_tag_index: dict[str, list[str]] = {}
|
||||
for doc_key, info in index.items():
|
||||
vn, fp = doc_key.split("::", 1) if "::" in doc_key else ("", "")
|
||||
if vn == vault_name:
|
||||
for t in info.get("tags", []):
|
||||
_tag_index.setdefault(t.lower(), []).append(fp)
|
||||
|
||||
if scope == "full":
|
||||
target = vault_root.resolve()
|
||||
effective_depth = depth if depth > 0 else 2
|
||||
else:
|
||||
target = resolve_safe_path(vault_root, path) if path else vault_root.resolve()
|
||||
effective_depth = depth
|
||||
|
||||
if not target.exists():
|
||||
raise ServiceError(
|
||||
f"Path not found: {path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"vault": vault_name, "path": path},
|
||||
)
|
||||
|
||||
focus_name = path.split("/")[-1] if path else vault_name
|
||||
focus_type = "directory" if path else "vault"
|
||||
focus_id = _add_node(focus_name, focus_type, path)
|
||||
|
||||
def _walk_dir(dir_path: Path, parent_id: str, current_depth: int) -> None:
|
||||
if current_depth > effective_depth:
|
||||
return
|
||||
try:
|
||||
for entry in sorted(dir_path.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower())):
|
||||
if hide_hidden and entry.name.startswith("."):
|
||||
continue
|
||||
rel = str(entry.relative_to(vault_root)).replace("\\", "/")
|
||||
|
||||
if tag and entry.is_file():
|
||||
file_tags = [t.lower() for t in _tag_index.get(rel, [])]
|
||||
if tag.lower() not in file_tags:
|
||||
continue
|
||||
|
||||
if entry.is_dir():
|
||||
did = _add_node(entry.name, "directory", rel)
|
||||
_add_edge(parent_id, did, "parent")
|
||||
if current_depth < effective_depth:
|
||||
_walk_dir(entry, did, current_depth + 1)
|
||||
elif entry.suffix.lower() in SUPPORTED_EXTENSIONS or entry.name.lower() in ("dockerfile", "makefile"):
|
||||
file_tags = _tag_index.get(rel, [])
|
||||
fid = _add_node(entry.name, "file", rel, entry.stat().st_size, tags=file_tags)
|
||||
_add_edge(parent_id, fid, "parent")
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
if target.is_dir():
|
||||
_walk_dir(target, focus_id, 0)
|
||||
elif target.is_file():
|
||||
_walk_dir(target.parent, focus_id, 0)
|
||||
|
||||
_add_wikilink_edges(nodes, edges, vault_name)
|
||||
|
||||
edge_counts: dict[str, dict[str, int]] = {}
|
||||
for node in nodes:
|
||||
edge_counts[node["id"]] = {"incoming": 0, "outgoing": 0}
|
||||
for edge in edges:
|
||||
if edge["relation"] in ("wikilink", "backlink"):
|
||||
src = edge["source"]
|
||||
tgt = edge["target"]
|
||||
if src in edge_counts:
|
||||
edge_counts[src]["outgoing"] += 1
|
||||
if tgt in edge_counts:
|
||||
edge_counts[tgt]["incoming"] += 1
|
||||
for node in nodes:
|
||||
counts = edge_counts.get(node["id"], {"incoming": 0, "outgoing": 0})
|
||||
node["incoming_count"] = counts["incoming"]
|
||||
node["outgoing_count"] = counts["outgoing"]
|
||||
|
||||
return {"vault": vault_name, "path": path, "scope": scope, "nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def _add_wikilink_edges(nodes: list[dict[str, Any]], edges: list[dict[str, Any]], vault_name: str) -> None:
|
||||
"""Add edges for wikilinks between markdown files in the current graph scope."""
|
||||
file_nodes = [n for n in nodes if n["type"] == "file" and n["path"].endswith(".md")]
|
||||
if len(file_nodes) < 2:
|
||||
return
|
||||
|
||||
path_to_id = {n["path"]: n["id"] for n in file_nodes}
|
||||
existing = {(e["source"], e["target"]) for e in edges}
|
||||
|
||||
for node in file_nodes:
|
||||
vault_data = index.get(vault_name)
|
||||
if not vault_data:
|
||||
continue
|
||||
file_entry = None
|
||||
for f in vault_data.get("files", []):
|
||||
if f["path"] == node["path"]:
|
||||
file_entry = f
|
||||
break
|
||||
if not file_entry:
|
||||
continue
|
||||
|
||||
content = file_entry.get("content", "")
|
||||
if not content:
|
||||
continue
|
||||
|
||||
for match in _WIKILINK_PATTERN.finditer(content):
|
||||
target = match.group(1).strip()
|
||||
target_lower = target.lower()
|
||||
if not target_lower.endswith(".md"):
|
||||
target_lower += ".md"
|
||||
|
||||
for target_path, target_id in path_to_id.items():
|
||||
if target_id == node["id"]:
|
||||
continue
|
||||
target_name = target_path.rsplit("/", 1)[-1].lower()
|
||||
if target_name == target_lower or target_path.lower() == target_lower:
|
||||
edge_key = tuple(sorted([node["id"], target_id]))
|
||||
if edge_key not in existing and (edge_key[1], edge_key[0]) not in existing:
|
||||
edges.append({"source": node["id"], "target": target_id, "relation": "wikilink"})
|
||||
existing.add((node["id"], target_id))
|
||||
break
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user