-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Add bracketing to lexer #68
Changes from 18 commits
dd87240
1ad6d60
a5b4df9
39d101b
81b49bc
a4c47dc
93e43c6
48612d4
e4842a6
3c2be6f
16b1c85
4e9ac06
a4997dc
06f671e
063634f
c7eab1f
859a144
937ef01
4806481
48e231b
7df8f65
44e1c3e
ebf139f
625451d
6cef983
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
module Brat.Lexer.Bracketed (BToken(..), brackets) where | ||
|
||
import Data.Bracket | ||
import Brat.Error (BracketErrMsg(..), Error(Err), ErrorMsg(..)) | ||
import Brat.FC | ||
import Brat.Lexer.Token | ||
import Bwd | ||
|
||
import Text.Megaparsec (PosState(..), SourcePos(..), TraversableStream(..), VisualStream(..)) | ||
import Text.Megaparsec.Pos (mkPos) | ||
|
||
opener :: Tok -> Maybe BracketType | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suggest
Then you can do Then zap both
Could also make |
||
opener LParen = Just Paren | ||
opener LSquare = Just Square | ||
opener LBrace = Just Brace | ||
opener _ = Nothing | ||
|
||
closer :: Tok -> Maybe BracketType | ||
closer RParen = Just Paren | ||
closer RSquare = Just Square | ||
closer RBrace = Just Brace | ||
closer _ = Nothing | ||
|
||
-- Well bracketed tokens | ||
data BToken | ||
= Bracketed FC BracketType [BToken] | ||
| FlatTok Token | ||
deriving (Eq, Ord) | ||
|
||
btokLen :: BToken -> Int | ||
btokLen (FlatTok tok) = length (show tok) | ||
btokLen (Bracketed _ _ bs) = sum (btokLen <$> bs) + 2 | ||
|
||
instance Show BToken where | ||
show (FlatTok t) = show t | ||
show (Bracketed _ b ts) = showOpen b ++ show ts ++ showClose b | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Or use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok, so I realize you can't |
||
|
||
instance VisualStream [BToken] where | ||
showTokens _ = concatMap show | ||
tokensLength _ = sum . fmap btokLen | ||
|
||
instance TraversableStream [BToken] where | ||
reachOffsetNoLine i pos = let fileName = sourceName (pstateSourcePos pos) | ||
(Pos line col, rest) = skipChars (i - pstateOffset pos + 1) (pstateInput pos) | ||
in pos | ||
{ pstateInput = rest | ||
, pstateOffset = max (pstateOffset pos) i | ||
, pstateSourcePos = SourcePos fileName (mkPos line) (mkPos col) | ||
} | ||
where | ||
skipChars :: Int -> [BToken] -> (Pos, [BToken]) | ||
skipChars 0 inp@(Bracketed fc _ _:_) = (start fc, inp) | ||
Check warning on line 52 in brat/Brat/Lexer/Bracketed.hs GitHub Actions / build
|
||
skipChars 0 inp@(FlatTok t:_) = (start (fc t), inp) | ||
skipChars i ((Bracketed fc b bts):rest) = | ||
let Pos closeLine closeCol = (end fc) | ||
Check warning on line 55 in brat/Brat/Lexer/Bracketed.hs GitHub Actions / hlint
|
||
closeFC = FC (Pos closeLine (closeCol - 1)) (Pos closeLine closeCol) | ||
in skipChars (i - 1) (bts ++ [FlatTok (Token closeFC (closeTok b))] ++ rest) | ||
skipChars i (FlatTok t:rest) | ||
| i >= tokenLen t = skipChars (i - tokenLen t) rest | ||
| otherwise = (start (fc t), FlatTok t:rest) | ||
|
||
closeTok Paren = RParen | ||
closeTok Square = RSquare | ||
closeTok Brace = RBrace | ||
|
||
eofErr :: FC -> BracketType -> Error | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think I suggested you curry this earlier and now I'm gonna make the reverse suggestion - sorry! But the "unit" of bracket, in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think I'm really getting it, but have uncurried anyway |
||
eofErr fc b = Err (Just fc) (BracketErr (EOFInBracket b)) | ||
|
||
openCloseMismatchErr :: (FC, BracketType) -> (FC, BracketType) -> Error | ||
openCloseMismatchErr open (fcClose, bClose) | ||
= Err (Just fcClose) (BracketErr (OpenCloseMismatch open bClose)) | ||
|
||
unexpectedCloseErr :: FC -> BracketType -> Error | ||
unexpectedCloseErr fc b = Err (Just fc) (BracketErr (UnexpectedClose b)) | ||
|
||
-- Parse between two brackets of the same type | ||
within :: (FC, BracketType) -- The nearest opening bracket to the left of us | ||
acl-cqc marked this conversation as resolved.
Show resolved
Hide resolved
|
||
-> Bwd BToken -- The tokens that we've passed since that open bracket | ||
-> [Token] -- The tokens to the right of us, unparsed | ||
-> Either Error (FC -- The location of the closing bracket | ||
,Bwd BToken -- The tokens between the open and close | ||
,[Token] -- Tokens after the closing bracket | ||
) | ||
within (openFC, b) _ [] = Left $ eofErr openFC b | ||
within ctx@(_, b) acc (t:ts) | ||
| Just b' <- closer (_tok t) = if b' == b | ||
then pure (fc t, acc, ts) | ||
else Left $ openCloseMismatchErr ctx (fc t, b') | ||
| Just b' <- opener (_tok t) = do | ||
let innerOpenFC = fc t | ||
(innerCloseFC, xs, ts) <- within (innerOpenFC, b') B0 ts | ||
let fc = spanFC innerOpenFC innerCloseFC | ||
within ctx (acc :< Bracketed fc b' (xs <>> [])) ts | ||
| otherwise = within ctx (acc :< FlatTok t) ts | ||
|
||
brackets :: [Token] -> Either Error [BToken] | ||
brackets ts = (<>> []) <$> bracketsWorker B0 ts | ||
where | ||
bracketsWorker :: Bwd BToken -> [Token] -> Either Error (Bwd BToken) | ||
bracketsWorker acc [] = pure acc | ||
bracketsWorker acc (t:ts) | ||
| Just b <- opener (_tok t) = do | ||
(closeFC, xs, ts) <- within (fc t, b) B0 ts | ||
let enclosingFC = spanFC (fc t) closeFC | ||
bracketsWorker (acc :< Bracketed enclosingFC b (xs <>> [])) ts | ||
| Just b <- closer (_tok t) = Left $ unexpectedCloseErr (fc t) b | ||
| otherwise = bracketsWorker (acc :< FlatTok t) ts |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is pretty much an OpenCloseMismatch, without an opening (or where the opening is beginning of file) - consider combining the two, i.e.
OpenCloseMismatch (Maybe (FC, BracketType)) BracketType
. Or not, the conceptual similarity could just be confusing, up to you...